diff --git a/.gitignore b/.gitignore index 51b9af9..b338b88 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,16 @@ go.work /log/ # Shield config files in the configs folder /configs/**/*.yaml +/configs/**/*.pem + +# ai config +.cursor/ +.claude/ +.cursorrules +.copilot/ +.chatgpt/ +.ai_history/ +.vector_cache/ +ai-debug.log +*.patch +*.diff \ No newline at end of file diff --git a/common/errcode/bussiness_error.go b/common/errcode/bussiness_error.go index 04ff3f6..6ab5999 100644 --- a/common/errcode/bussiness_error.go +++ b/common/errcode/bussiness_error.go @@ -16,6 +16,12 @@ var ( // ErrFoundTargetFailed define variable to returned when the specific database table cannot be identified using the provided token info. ErrFoundTargetFailed = newError(40004, "found target table by token failed") + // ErrSubTargetRepeat define variable to indicates subscription target already exist in list + ErrSubTargetRepeat = newError(40005, "subscription target already exist in list") + // ErrSubTargetNotFound define variable to indicates can not find measurement by subscription target + ErrSubTargetNotFound = newError(40006, "found measuremnet by subscription target failed") + // ErrCancelSubTargetMissing define variable to indicates cancel a not exist subscription target + ErrCancelSubTargetMissing = newError(40007, "cancel a not exist subscription target") // ErrDBQueryFailed define variable to represents a generic failure during a PostgreSQL SELECT or SCAN operation. ErrDBQueryFailed = newError(50001, "query postgres database data failed") @@ -40,4 +46,10 @@ var ( // ErrCacheQueryFailed define variable to indicates query cached data by token failed. ErrCacheQueryFailed = newError(60003, "query cached data by token failed") + + // ErrTaskNotFound indicates the async task with the given ID does not exist. + ErrTaskNotFound = newError(40008, "async task not found") + + // ErrTaskCannotCancel indicates the task is already running or completed and cannot be cancelled. + ErrTaskCannotCancel = newError(40009, "task cannot be cancelled, already running or completed") ) diff --git a/common/errcode/error.go b/common/errcode/error.go index 18eca89..1d12276 100644 --- a/common/errcode/error.go +++ b/common/errcode/error.go @@ -139,10 +139,10 @@ func (e *AppError) SetMsg(msg string) *AppError { } type formattedErr struct { - Code int `json:"code"` - Msg string `json:"msg"` - Cause interface{} `json:"cause"` - Occurred string `json:"occurred"` + Code int `json:"code"` + Msg string `json:"msg"` + Cause any `json:"cause"` + Occurred string `json:"occurred"` } // toStructuredError define func convert AppError to structured error for better readability diff --git a/common/event_action_errors.go b/common/event_action_errors.go new file mode 100644 index 0000000..3df4a39 --- /dev/null +++ b/common/event_action_errors.go @@ -0,0 +1,10 @@ +// Package common define common error variables +package common + +import "errors" + +// ErrUnknowEventActionCommand define error of unknown event action command +var ErrUnknowEventActionCommand = errors.New("unknown action command") + +// ErrExecEventActionFailed define error of execute event action failed +var ErrExecEventActionFailed = errors.New("exec event action func failed") diff --git a/constants/error.go b/common/uuid_errors.go similarity index 97% rename from constants/error.go rename to common/uuid_errors.go index da0b91e..bca1b32 100644 --- a/constants/error.go +++ b/common/uuid_errors.go @@ -1,5 +1,5 @@ -// Package constants define constant variable -package constants +// Package common define common error variables +package common import "errors" diff --git a/config/anchor_param_config.go b/config/anchor_param_config.go index 018aca9..b5e201c 100644 --- a/config/anchor_param_config.go +++ b/config/anchor_param_config.go @@ -42,12 +42,13 @@ var baseCurrentFunc = func(archorValue float64, args ...float64) float64 { } // SelectAnchorCalculateFuncAndParams define select anchor func and anchor calculate value by component type 、 anchor name and component data -func SelectAnchorCalculateFuncAndParams(componentType int, anchorName string, componentData map[string]interface{}) (func(archorValue float64, args ...float64) float64, []float64) { +func SelectAnchorCalculateFuncAndParams(componentType int, anchorName string, componentData map[string]any) (func(archorValue float64, args ...float64) float64, []float64) { if componentType == constants.DemoType { - if anchorName == "voltage" { + switch anchorName { + case "voltage": resistance := componentData["resistance"].(float64) return baseVoltageFunc, []float64{resistance} - } else if anchorName == "current" { + case "current": resistance := componentData["resistance"].(float64) return baseCurrentFunc, []float64{resistance} } diff --git a/config/config.go b/config/config.go index 21e4129..a0ddc7f 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" + "time" "github.com/spf13/viper" ) @@ -19,6 +20,21 @@ type ServiceConfig struct { ServiceAddr string `mapstructure:"service_addr"` ServiceName string `mapstructure:"service_name"` SecretKey string `mapstructure:"secret_key"` + DeployEnv string `mapstructure:"deploy_env"` +} + +// RabbitMQConfig define config struct of RabbitMQ config +type RabbitMQConfig struct { + CACertPath string `mapstructure:"ca_cert_path"` + ClientKeyPath string `mapstructure:"client_key_path"` + ClientKeyPassword string `mapstructure:"client_key_password"` + ClientCertPath string `mapstructure:"client_cert_path"` + InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"` + ServerName string `mapstructure:"server_name"` + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` } // KafkaConfig define config struct of kafka config @@ -40,24 +56,33 @@ type PostgresConfig struct { Password string `mapstructure:"password"` } +// LokiConfig define config struct of loki direct-push (used in development mode) +type LokiConfig struct { + Endpoint string `mapstructure:"endpoint"` // empty disables direct push + Labels map[string]string `mapstructure:"labels"` +} + // LoggerConfig define config struct of zap logger config type LoggerConfig struct { - Mode string `mapstructure:"mode"` - Level string `mapstructure:"level"` - FilePath string `mapstructure:"filepath"` - MaxSize int `mapstructure:"maxsize"` - MaxBackups int `mapstructure:"maxbackups"` - MaxAge int `mapstructure:"maxage"` - Compress bool `mapstructure:"compress"` + Mode string `mapstructure:"mode"` + Level string `mapstructure:"level"` + FilePath string `mapstructure:"filepath"` // empty disables file rotation in container modes + MaxSize int `mapstructure:"maxsize"` + MaxBackups int `mapstructure:"maxbackups"` + MaxAge int `mapstructure:"maxage"` + Compress bool `mapstructure:"compress"` + Loki LokiConfig `mapstructure:"loki"` } // RedisConfig define config struct of redis config type RedisConfig struct { - Addr string `mapstructure:"addr"` - Password string `mapstructure:"password"` - DB int `mapstructure:"db"` - PoolSize int `mapstructure:"poolsize"` - Timeout int `mapstructure:"timeout"` + Addr string `mapstructure:"addr"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` + PoolSize int `mapstructure:"poolsize"` + DialTimeout int `mapstructure:"dial_timeout"` + ReadTimeout int `mapstructure:"read_timeout"` + WriteTimeout int `mapstructure:"write_timeout"` } // AntsConfig define config struct of ants pool config @@ -74,18 +99,37 @@ type DataRTConfig struct { Method string `mapstructure:"polling_api_method"` } +// OtelConfig define config struct of OpenTelemetry tracing +type OtelConfig struct { + Endpoint string `mapstructure:"endpoint"` // e.g. "localhost:4318" + Insecure bool `mapstructure:"insecure"` +} + +// AsyncTaskConfig define config struct of asynchronous task system +type AsyncTaskConfig struct { + WorkerPoolSize int `mapstructure:"worker_pool_size"` + QueueConsumerCount int `mapstructure:"queue_consumer_count"` + MaxRetryCount int `mapstructure:"max_retry_count"` + RetryInitialDelay time.Duration `mapstructure:"retry_initial_delay"` + RetryMaxDelay time.Duration `mapstructure:"retry_max_delay"` + HealthCheckInterval time.Duration `mapstructure:"health_check_interval"` +} + // ModelRTConfig define config struct of model runtime server type ModelRTConfig struct { BaseConfig `mapstructure:"base"` ServiceConfig `mapstructure:"service"` PostgresConfig `mapstructure:"postgres"` + RabbitMQConfig `mapstructure:"rabbitmq"` KafkaConfig `mapstructure:"kafka"` LoggerConfig `mapstructure:"logger"` AntsConfig `mapstructure:"ants"` DataRTConfig `mapstructure:"dataRT"` LockerRedisConfig RedisConfig `mapstructure:"locker_redis"` StorageRedisConfig RedisConfig `mapstructure:"storage_redis"` - PostgresDBURI string `mapstructure:"-"` + AsyncTaskConfig AsyncTaskConfig `mapstructure:"async_task"` + OtelConfig OtelConfig `mapstructure:"otel"` + PostgresDBURI string `mapstructure:"-"` } // ReadAndInitConfig return modelRT project config struct @@ -101,6 +145,9 @@ func ReadAndInitConfig(configDir, configName, configType string) (modelRTConfig panic(err) } + config.BindEnv("postgres.password", "POSTGRES_PASSWORD") + config.BindEnv("service.secret_key", "SERVICE_SECRET_KEY") + if err := config.Unmarshal(&modelRTConfig); err != nil { panic(fmt.Sprintf("unmarshal modelRT config failed:%s\n", err.Error())) } diff --git a/constants/attribute_business_code.go b/constants/attribute_business_code.go deleted file mode 100644 index 487b338..0000000 --- a/constants/attribute_business_code.go +++ /dev/null @@ -1,17 +0,0 @@ -// Package constants define constant variable -package constants - -const ( - // CodeSuccess define constant to indicates that the API was successfully processed - CodeSuccess = 20000 - // CodeInvalidParamFailed define constant to indicates request parameter parsing failed - CodeInvalidParamFailed = 40001 - // CodeDBQueryFailed define constant to indicates database query operation failed - CodeDBQueryFailed = 50001 - // CodeDBUpdateailed define constant to indicates database update operation failed - CodeDBUpdateailed = 50002 - // CodeRedisQueryFailed define constant to indicates redis query operation failed - CodeRedisQueryFailed = 60001 - // CodeRedisUpdateFailed define constant to indicates redis update operation failed - CodeRedisUpdateFailed = 60002 -) diff --git a/constants/business_code.go b/constants/business_code.go new file mode 100644 index 0000000..aa18e0e --- /dev/null +++ b/constants/business_code.go @@ -0,0 +1,31 @@ +// Package constants define constant variable +package constants + +const ( + // CodeSuccess define constant to indicates that the API was successfully processed + CodeSuccess = 20000 + // CodeInvalidParamFailed define constant to indicates request parameter parsing failed + CodeInvalidParamFailed = 40001 + // CodeFoundTargetFailed define variable to returned when the specific database table cannot be identified using the provided token info. + CodeFoundTargetFailed = 40004 + // CodeSubTargetRepeat define variable to indicates subscription target already exist in list + CodeSubTargetRepeat = 40005 + // CodeSubTargetNotFound define variable to indicates can not find measurement by subscription target + CodeSubTargetNotFound = 40006 + // CodeCancelSubTargetMissing define variable to indicates cancel a not exist subscription target + CodeCancelSubTargetMissing = 40007 + // CodeUpdateSubTargetMissing define variable to indicates update a not exist subscription target + CodeUpdateSubTargetMissing = 40008 + // CodeAppendSubTargetMissing define variable to indicates append a not exist subscription target + CodeAppendSubTargetMissing = 40009 + // CodeUnsupportSubOperation define variable to indicates append a not exist subscription target + CodeUnsupportSubOperation = 40010 + // CodeDBQueryFailed define constant to indicates database query operation failed + CodeDBQueryFailed = 50001 + // CodeDBUpdateailed define constant to indicates database update operation failed + CodeDBUpdateailed = 50002 + // CodeRedisQueryFailed define constant to indicates redis query operation failed + CodeRedisQueryFailed = 60001 + // CodeRedisUpdateFailed define constant to indicates redis update operation failed + CodeRedisUpdateFailed = 60002 +) diff --git a/constants/deploy_mode.go b/constants/deploy_mode.go new file mode 100644 index 0000000..2a5addf --- /dev/null +++ b/constants/deploy_mode.go @@ -0,0 +1,11 @@ +// Package constants define constant variable +package constants + +const ( + // DevelopmentDeployMode define development operator environment for modelRT project + DevelopmentDeployMode = "development" + // DebugDeployMode define debug operator environment for modelRT project + DebugDeployMode = "debug" + // ProductionDeployMode define production operator environment for modelRT project + ProductionDeployMode = "production" +) diff --git a/constants/event.go b/constants/event.go index 67bb63a..293ca25 100644 --- a/constants/event.go +++ b/constants/event.go @@ -1,31 +1,97 @@ // Package constants define constant variable package constants +// EvenvtType define event type +type EvenvtType int + const ( - // TIBreachTriggerType define out of bounds type constant - TIBreachTriggerType = "trigger" + // EventGeneralHard define gereral hard event type + EventGeneralHard EvenvtType = iota + // EventGeneralPlatformSoft define gereral platform soft event type + EventGeneralPlatformSoft + // EventGeneralApplicationSoft define gereral application soft event type + EventGeneralApplicationSoft + // EventWarnHard define warn hard event type + EventWarnHard + // EventWarnPlatformSoft define warn platform soft event type + EventWarnPlatformSoft + // EventWarnApplicationSoft define warn application soft event type + EventWarnApplicationSoft + // EventCriticalHard define critical hard event type + EventCriticalHard + // EventCriticalPlatformSoft define critical platform soft event type + EventCriticalPlatformSoft + // EventCriticalApplicationSoft define critical application soft event type + EventCriticalApplicationSoft +) + +// IsGeneral define fucn to check event type is general +func IsGeneral(eventType EvenvtType) bool { + return eventType < 3 +} + +// IsWarning define fucn to check event type is warn +func IsWarning(eventType EvenvtType) bool { + return eventType >= 3 && eventType <= 5 +} + +// IsCritical define fucn to check event type is critical +func IsCritical(eventType EvenvtType) bool { + return eventType >= 6 +} + +const ( + // EventFromStation define event from station type + EventFromStation = "station" + // EventFromPlatform define event from platform type + EventFromPlatform = "platform" + // EventFromOthers define event from others type + EventFromOthers = "others" ) const ( - // TelemetryUpLimit define telemetry upper limit - TelemetryUpLimit = "up" - // TelemetryUpUpLimit define telemetry upper upper limit - TelemetryUpUpLimit = "upup" - - // TelemetryDownLimit define telemetry limit - TelemetryDownLimit = "down" - // TelemetryDownDownLimit define telemetry lower lower limit - TelemetryDownDownLimit = "downdown" + // EventStatusHappended define status for event record when event just happened, no data attached yet + EventStatusHappended = iota + // EventStatusDataAttached define status for event record when event data attached, ready to be sent + EventStatusDataAttached + // EventStatusReported define status for event record when event reported to downstream, no matter it's successful or failed + EventStatusReported + // EventStatusConfirmed define status for event record when event confirmed by operator or CIM + EventStatusConfirmed + // EventStatusClosed define status for event record when event closed due to condition recovery or manual close + EventStatusClosed ) const ( - // TelesignalRaising define telesignal raising edge - TelesignalRaising = "raising" - // TelesignalFalling define telesignal falling edge - TelesignalFalling = "falling" + // EventExchangeName define exchange name for event alarm message + EventExchangeName = "event-exchange" + // EventDeadExchangeName define dead letter exchange name for event alarm message + EventDeadExchangeName = "event-dead-letter-exchange" ) const ( - // MinBreachCount define min breach count of real time data - MinBreachCount = 10 + // EventUpDownRoutingKey define routing key for up or down limit event alarm message + EventUpDownRoutingKey = "event.#" + // EventUpDownDeadRoutingKey define dead letter routing key for up or down limit event alarm message + EventUpDownDeadRoutingKey = "event.#" + // EventUpDownQueueName define queue name for up or down limit event alarm message + EventUpDownQueueName = "event-up-down-queue" + // EventUpDownDeadQueueName define dead letter queue name for event alarm message + EventUpDownDeadQueueName = "event-dead-letter-queue" +) + +const ( + // EventGeneralUpDownLimitCategroy define category for general up and down limit event + EventGeneralUpDownLimitCategroy = "event.general.updown.limit" + // EventWarnUpDownLimitCategroy define category for warn up and down limit event + EventWarnUpDownLimitCategroy = "event.warn.updown.limit" + // EventCriticalUpDownLimitCategroy define category for critical up and down limit event + EventCriticalUpDownLimitCategroy = "event.critical.updown.limit" +) + +const ( + // EventTaskGeneralTestCategory define category for test task event + EventTaskGeneralTestCategory = "event.general.task.test" + // EventTaskGeneralTopologyAnalyzeCategory define category for topology analyze task event + EventTaskGeneralTopologyAnalyzeCategory = "event.general.task.topology_analyze" ) diff --git a/constants/message.go b/constants/message.go new file mode 100644 index 0000000..7b6fb1f --- /dev/null +++ b/constants/message.go @@ -0,0 +1,33 @@ +// Package constants define constant variable +package constants + +const ( + // MessageExchangeName define exchange name for message + MessageExchangeName = "message-exchange" + // MessageDeadExchangeName define dead letter exchange name for message + MessageDeadExchangeName = "message-dead-letter-exchange" +) + +const ( + // MessageRoutingKey define binding routing key pattern for the message queue (matches all message.* categories) + MessageRoutingKey = "message.#" + // MessageDeadRoutingKey define binding routing key for the message dead letter queue + MessageDeadRoutingKey = "#" + // MessageQueueName define queue name for message + MessageQueueName = "message-queue" + // MessageDeadQueueName define dead letter queue name for message + MessageDeadQueueName = "message-dead-letter-queue" +) + +const ( + // MessageTaskSubmittedCategory define category for task submitted message + MessageTaskSubmittedCategory = "message.task.submitted" + // MessageTaskRunningCategory define category for task running message + MessageTaskRunningCategory = "message.task.running" + // MessageTaskCompletedCategory define category for task completed message + MessageTaskCompletedCategory = "message.task.completed" + // MessageTaskFailedCategory define category for task failed message + MessageTaskFailedCategory = "message.task.failed" + // MessageTaskCancelledCategory define category for task cancelled message + MessageTaskCancelledCategory = "message.task.cancelled" +) diff --git a/constants/subscription_business_code.go b/constants/rtdata_subscription.go similarity index 74% rename from constants/subscription_business_code.go rename to constants/rtdata_subscription.go index 4b202bb..e14a3da 100644 --- a/constants/subscription_business_code.go +++ b/constants/rtdata_subscription.go @@ -12,29 +12,6 @@ const ( SubUpdateAction string = "update" ) -// 定义状态常量 -// TODO 从4位格式修改为5位格式 -const ( - // SubSuccessCode define subscription success code - SubSuccessCode = "1001" - // SubFailedCode define subscription failed code - SubFailedCode = "1002" - // RTDSuccessCode define real time data return success code - RTDSuccessCode = "1003" - // RTDFailedCode define real time data return failed code - RTDFailedCode = "1004" - // CancelSubSuccessCode define cancel subscription success code - CancelSubSuccessCode = "1005" - // CancelSubFailedCode define cancel subscription failed code - CancelSubFailedCode = "1006" - // SubRepeatCode define subscription repeat code - SubRepeatCode = "1007" - // UpdateSubSuccessCode define update subscription success code - UpdateSubSuccessCode = "1008" - // UpdateSubFailedCode define update subscription failed code - UpdateSubFailedCode = "1009" -) - const ( // SysCtrlPrefix define to indicates the prefix for all system control directives,facilitating unified parsing within the sendDataStream goroutine SysCtrlPrefix = "SYS_CTRL_" diff --git a/constants/task.go b/constants/task.go new file mode 100644 index 0000000..bcdd09b --- /dev/null +++ b/constants/task.go @@ -0,0 +1,54 @@ +// Package constants defines task-related constants for the async task system +package constants + +import "time" + +// Task priority levels +const ( + // TaskPriorityDefault is the default priority level for tasks + TaskPriorityDefault = 5 + // TaskPriorityHigh represents high priority tasks + TaskPriorityHigh = 10 + // TaskPriorityLow represents low priority tasks + TaskPriorityLow = 1 +) + +// Task queue configuration +const ( + // TaskExchangeName is the name of the exchange for task routing + TaskExchangeName = "modelrt.tasks.exchange" + // TaskQueueName is the name of the main task queue + TaskQueueName = "modelrt.tasks.queue" + // TaskRoutingKey is the routing key for task messages + TaskRoutingKey = "modelrt.task" +) + +// Task message settings +const ( + // TaskMaxPriority is the maximum priority level for tasks (0-10) + TaskMaxPriority = 10 + // TaskDefaultMessageTTL is the default time-to-live for task messages (24 hours) + TaskDefaultMessageTTL = 24 * time.Hour +) + +// Task retry settings +const ( + // TaskRetryMaxDefault is the default maximum number of retry attempts + TaskRetryMaxDefault = 3 + // TaskRetryInitialDelayDefault is the default initial delay for exponential backoff + TaskRetryInitialDelayDefault = 1 * time.Second + // TaskRetryMaxDelayDefault is the default maximum delay for exponential backoff + TaskRetryMaxDelayDefault = 5 * time.Minute + // TaskRetryRandomFactorDefault is the default random factor for jitter (10%) + TaskRetryRandomFactorDefault = 0.1 + // TaskRetryFixedDelayDefault is the default delay for fixed retry strategy + TaskRetryFixedDelayDefault = 5 * time.Second +) + +// Test task settings +const ( + // TestTaskSleepDurationDefault is the default sleep duration for test tasks (60 seconds) + TestTaskSleepDurationDefault = 60 + // TestTaskSleepDurationMax is the maximum allowed sleep duration for test tasks (1 hour) + TestTaskSleepDurationMax = 3600 +) diff --git a/constants/telemetry_imit.go b/constants/telemetry_imit.go new file mode 100644 index 0000000..67bb63a --- /dev/null +++ b/constants/telemetry_imit.go @@ -0,0 +1,31 @@ +// Package constants define constant variable +package constants + +const ( + // TIBreachTriggerType define out of bounds type constant + TIBreachTriggerType = "trigger" +) + +const ( + // TelemetryUpLimit define telemetry upper limit + TelemetryUpLimit = "up" + // TelemetryUpUpLimit define telemetry upper upper limit + TelemetryUpUpLimit = "upup" + + // TelemetryDownLimit define telemetry limit + TelemetryDownLimit = "down" + // TelemetryDownDownLimit define telemetry lower lower limit + TelemetryDownDownLimit = "downdown" +) + +const ( + // TelesignalRaising define telesignal raising edge + TelesignalRaising = "raising" + // TelesignalFalling define telesignal falling edge + TelesignalFalling = "falling" +) + +const ( + // MinBreachCount define min breach count of real time data + MinBreachCount = 10 +) diff --git a/constants/trace.go b/constants/trace.go index 14f52ed..e5e595a 100644 --- a/constants/trace.go +++ b/constants/trace.go @@ -1,9 +1,21 @@ // Package constants define constant variable package constants -// Assuming the B3 specification +// Internal context keys for trace values set by StartTrace middleware. +// These are gin/stdlib context keys only — actual W3C header propagation +// (traceparent / tracestate) is handled automatically by the OTel propagator. const ( - HeaderTraceID = "X-B3-TraceId" - HeaderSpanID = "X-B3-SpanId" - HeaderParentSpanID = "X-B3-ParentSpanId" + HeaderTraceID = "trace-id" + HeaderSpanID = "span-id" + HeaderParentSpanID = "parent-span-id" +) + +// traceCtxKey is an unexported type for context keys to avoid collisions with other packages. +type traceCtxKey string + +// Typed context keys for trace values — use these with context.WithValue / ctx.Value. +var ( + CtxKeyTraceID = traceCtxKey(HeaderTraceID) + CtxKeySpanID = traceCtxKey(HeaderSpanID) + CtxKeyParentSpanID = traceCtxKey(HeaderParentSpanID) ) diff --git a/database/async_task_extended.go b/database/async_task_extended.go new file mode 100644 index 0000000..8d5849c --- /dev/null +++ b/database/async_task_extended.go @@ -0,0 +1,228 @@ +// Package database define database operation functions +package database + +import ( + "context" + "time" + + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// UpdateTaskStarted updates task start time and status to running +func UpdateTaskStarted(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, startedAt int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusRunning, + "started_at": startedAt, + }) + + return result.Error +} + +// UpdateTaskRetryInfo updates task retry information +func UpdateTaskRetryInfo(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, retryCount int, nextRetryTime int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + updateData := map[string]any{ + "retry_count": retryCount, + } + if nextRetryTime <= 0 { + updateData["next_retry_time"] = nil + } else { + updateData["next_retry_time"] = nextRetryTime + } + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(updateData) + + return result.Error +} + +// UpdateTaskErrorInfo updates task error information +func UpdateTaskErrorInfo(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, errorMsg, stackTrace string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "failure_reason": errorMsg, + "stack_trace": stackTrace, + "status": orm.AsyncTaskStatusFailed, + }) + + return result.Error +} + +// UpdateTaskExecutionTime updates task execution time +func UpdateTaskExecutionTime(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, executionTime int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("execution_time", executionTime) + + return result.Error +} + +// UpdateTaskWorkerID updates the worker ID that is processing the task +func UpdateTaskWorkerID(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, workerID string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("worker_id", workerID) + + return result.Error +} + +// UpdateTaskPriority updates task priority +func UpdateTaskPriority(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, priority int) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("priority", priority) + + return result.Error +} + +// UpdateTaskQueueName updates task queue name +func UpdateTaskQueueName(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, queueName string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("queue_name", queueName) + + return result.Error +} + +// UpdateTaskCreatedBy updates task creator information +func UpdateTaskCreatedBy(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, createdBy string) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("created_by", createdBy) + + return result.Error +} + +// UpdateTaskResultWithMetrics updates task result with execution metrics +func UpdateTaskResultWithMetrics(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, executionTime int64, memoryUsage *int64, cpuUsage *float64, retryCount int, completedAt int64) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTaskResult{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "execution_time": executionTime, + "memory_usage": memoryUsage, + "cpu_usage": cpuUsage, + "retry_count": retryCount, + "completed_at": completedAt, + }) + + return result.Error +} + +// GetTasksForRetry retrieves tasks that are due for retry +func GetTasksForRetry(ctx context.Context, tx *gorm.DB, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + now := time.Now().Unix() + result := tx.WithContext(cancelCtx). + Where("status = ? AND next_retry_time IS NOT NULL AND next_retry_time <= ?", orm.AsyncTaskStatusFailed, now). + Order("next_retry_time ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// GetTasksByPriority retrieves tasks by priority order +func GetTasksByPriority(ctx context.Context, tx *gorm.DB, status orm.AsyncTaskStatus, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("status = ?", status). + Order("priority DESC, created_at ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// GetTasksByWorkerID retrieves tasks being processed by a specific worker +func GetTasksByWorkerID(ctx context.Context, tx *gorm.DB, workerID string) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("worker_id = ? AND status = ?", workerID, orm.AsyncTaskStatusRunning). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// CleanupStaleTasks marks tasks as failed if they have been running for too long +func CleanupStaleTasks(ctx context.Context, tx *gorm.DB, timeoutSeconds int64) (int64, error) { + cancelCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + threshold := time.Now().Unix() - timeoutSeconds + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("status = ? AND started_at IS NOT NULL AND started_at < ?", orm.AsyncTaskStatusRunning, threshold). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusFailed, + "failure_reason": "task timeout", + "finished_at": time.Now().Unix(), + }) + + return result.RowsAffected, result.Error +} \ No newline at end of file diff --git a/database/async_task_operations.go b/database/async_task_operations.go new file mode 100644 index 0000000..fce150b --- /dev/null +++ b/database/async_task_operations.go @@ -0,0 +1,323 @@ +// Package database define database operation functions +package database + +import ( + "context" + "time" + + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// CreateAsyncTask creates a new async task in the database +func CreateAsyncTask(ctx context.Context, tx *gorm.DB, taskType orm.AsyncTaskType, params orm.JSONMap) (*orm.AsyncTask, error) { + taskID, err := uuid.NewV4() + if err != nil { + return nil, err + } + + task := &orm.AsyncTask{ + TaskID: taskID, + TaskType: taskType, + Status: orm.AsyncTaskStatusSubmitted, + Params: params, + CreatedAt: time.Now().Unix(), + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx).Create(task) + if result.Error != nil { + return nil, result.Error + } + + return task, nil +} + +// GetAsyncTaskByID retrieves an async task by its ID +func GetAsyncTaskByID(ctx context.Context, tx *gorm.DB, taskID uuid.UUID) (*orm.AsyncTask, error) { + var task orm.AsyncTask + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + Clauses(clause.Locking{Strength: "UPDATE"}). + First(&task) + + if result.Error != nil { + return nil, result.Error + } + + return &task, nil +} + +// GetAsyncTasksByIDs retrieves multiple async tasks by their IDs +func GetAsyncTasksByIDs(ctx context.Context, tx *gorm.DB, taskIDs []uuid.UUID) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + if len(taskIDs) == 0 { + return tasks, nil + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id IN ?", taskIDs). + Clauses(clause.Locking{Strength: "UPDATE"}). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// UpdateAsyncTaskStatus updates the status of an async task +func UpdateAsyncTaskStatus(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, status orm.AsyncTaskStatus) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("status", status) + + return result.Error +} + +// UpdateAsyncTaskProgress updates the progress of an async task +func UpdateAsyncTaskProgress(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, progress int) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Update("progress", progress) + + return result.Error +} + +// CompleteAsyncTask marks an async task as completed with timestamp +func CompleteAsyncTask(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, timestamp int64) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusCompleted, + "finished_at": timestamp, + "progress": 100, + }) + + return result.Error +} + +// FailAsyncTask marks an async task as failed with timestamp +func FailAsyncTask(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, timestamp int64) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.AsyncTask{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "status": orm.AsyncTaskStatusFailed, + "finished_at": timestamp, + }) + + return result.Error +} + +// CreateAsyncTaskResult creates a result record for an async task +func CreateAsyncTaskResult(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, result orm.JSONMap) error { + taskResult := &orm.AsyncTaskResult{ + TaskID: taskID, + Result: result, + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + resultOp := tx.WithContext(cancelCtx).Create(taskResult) + return resultOp.Error +} + +// UpdateAsyncTaskResultWithError upserts a task result with error information. +func UpdateAsyncTaskResultWithError(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, code int, message string, detail orm.JSONMap) error { + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if err := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + FirstOrCreate(&orm.AsyncTaskResult{TaskID: taskID}).Error; err != nil { + return err + } + + return tx.WithContext(cancelCtx). + Model(&orm.AsyncTaskResult{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "error_code": code, + "error_message": message, + "error_detail": detail, + "result": nil, + }).Error +} + +// UpdateAsyncTaskResultWithSuccess updates a task result with success information +func UpdateAsyncTaskResultWithSuccess(ctx context.Context, tx *gorm.DB, taskID uuid.UUID, result orm.JSONMap) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + // First try to update existing record, if not found create new one + existingResult := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + FirstOrCreate(&orm.AsyncTaskResult{TaskID: taskID}) + + if existingResult.Error != nil { + return existingResult.Error + } + + // Update with success information + updateResult := tx.WithContext(cancelCtx). + Model(&orm.AsyncTaskResult{}). + Where("task_id = ?", taskID). + Updates(map[string]any{ + "result": result, + "error_code": nil, + "error_message": nil, + "error_detail": nil, + }) + + return updateResult.Error +} + +// GetAsyncTaskResult retrieves the result of an async task +func GetAsyncTaskResult(ctx context.Context, tx *gorm.DB, taskID uuid.UUID) (*orm.AsyncTaskResult, error) { + var taskResult orm.AsyncTaskResult + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id = ?", taskID). + First(&taskResult) + + if result.Error != nil { + if result.Error == gorm.ErrRecordNotFound { + return nil, nil + } + return nil, result.Error + } + + return &taskResult, nil +} + +// GetAsyncTaskResults retrieves multiple task results by task IDs +func GetAsyncTaskResults(ctx context.Context, tx *gorm.DB, taskIDs []uuid.UUID) ([]orm.AsyncTaskResult, error) { + var taskResults []orm.AsyncTaskResult + + if len(taskIDs) == 0 { + return taskResults, nil + } + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("task_id IN ?", taskIDs). + Find(&taskResults) + + if result.Error != nil { + return nil, result.Error + } + + return taskResults, nil +} + +// GetPendingTasks retrieves pending tasks (submitted but not yet running/completed) +func GetPendingTasks(ctx context.Context, tx *gorm.DB, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("status = ?", orm.AsyncTaskStatusSubmitted). + Order("created_at ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// GetTasksByStatus retrieves tasks by status +func GetTasksByStatus(ctx context.Context, tx *gorm.DB, status orm.AsyncTaskStatus, limit int) ([]orm.AsyncTask, error) { + var tasks []orm.AsyncTask + + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("status = ?", status). + Order("created_at ASC"). + Limit(limit). + Find(&tasks) + + if result.Error != nil { + return nil, result.Error + } + + return tasks, nil +} + +// DeleteOldTasks deletes tasks older than the specified timestamp +func DeleteOldTasks(ctx context.Context, tx *gorm.DB, olderThan int64) error { + // ctx timeout judgment + cancelCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + // First delete task results + result := tx.WithContext(cancelCtx). + Where("task_id IN (SELECT task_id FROM async_task WHERE created_at < ?)", olderThan). + Delete(&orm.AsyncTaskResult{}) + + if result.Error != nil { + return result.Error + } + + // Then delete tasks + result = tx.WithContext(cancelCtx). + Where("created_at < ?", olderThan). + Delete(&orm.AsyncTask{}) + + return result.Error +} \ No newline at end of file diff --git a/database/create_component.go b/database/create_component.go index 1c288c0..3d304a5 100644 --- a/database/create_component.go +++ b/database/create_component.go @@ -33,7 +33,7 @@ func CreateComponentIntoDB(ctx context.Context, tx *gorm.DB, componentInfo netwo Name: componentInfo.Name, Context: componentInfo.Context, Op: componentInfo.Op, - Ts: time.Now(), + TS: time.Now(), } result := tx.WithContext(cancelCtx).Create(&component) diff --git a/database/create_measurement.go b/database/create_measurement.go index 4085c94..30d6ad6 100644 --- a/database/create_measurement.go +++ b/database/create_measurement.go @@ -35,7 +35,7 @@ func CreateMeasurement(ctx context.Context, tx *gorm.DB, measurementInfo network BayUUID: globalUUID, ComponentUUID: globalUUID, Op: -1, - Ts: time.Now(), + TS: time.Now(), } result := tx.WithContext(cancelCtx).Create(&measurement) diff --git a/database/fill_identity_token_model.go b/database/fill_identity_token_model.go index 78a0e48..5f0fdd5 100644 --- a/database/fill_identity_token_model.go +++ b/database/fill_identity_token_model.go @@ -53,7 +53,8 @@ func FillingLongTokenModel(ctx context.Context, tx *gorm.DB, identModel *model.L func ParseDataIdentifierToken(ctx context.Context, tx *gorm.DB, identToken string) (model.IndentityTokenModelInterface, error) { identSlice := strings.Split(identToken, ".") identSliceLen := len(identSlice) - if identSliceLen == 4 { + switch identSliceLen { + case 4: // token1.token2.token3.token4.token7 shortIndentModel := &model.ShortIdentityTokenModel{ GridTag: identSlice[0], @@ -67,7 +68,7 @@ func ParseDataIdentifierToken(ctx context.Context, tx *gorm.DB, identToken strin return nil, err } return shortIndentModel, nil - } else if identSliceLen == 7 { + case 7: // token1.token2.token3.token4.token5.token6.token7 longIndentModel := &model.LongIdentityTokenModel{ GridTag: identSlice[0], diff --git a/database/filling_attr_model_info.go b/database/filling_attr_model_info.go index 408f7f7..6debf21 100644 --- a/database/filling_attr_model_info.go +++ b/database/filling_attr_model_info.go @@ -19,7 +19,8 @@ func ParseAttrToken(ctx context.Context, tx *gorm.DB, attrToken, clientToken str attrSlice := strings.Split(attrToken, ".") attrLen := len(attrSlice) - if attrLen == 4 { + switch attrLen { + case 4: short := &model.ShortAttrInfo{ AttrGroupName: attrSlice[2], AttrKey: attrSlice[3], @@ -35,7 +36,7 @@ func ParseAttrToken(ctx context.Context, tx *gorm.DB, attrToken, clientToken str } short.AttrValue = attrValue return short, nil - } else if attrLen == 7 { + case 7: long := &model.LongAttrInfo{ AttrGroupName: attrSlice[5], AttrKey: attrSlice[6], diff --git a/database/postgres_init.go b/database/postgres_init.go index a80d9fc..c2e0ed0 100644 --- a/database/postgres_init.go +++ b/database/postgres_init.go @@ -4,9 +4,9 @@ package database import ( "context" "sync" - "time" "modelRT/logger" + "modelRT/orm" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -15,15 +15,11 @@ import ( var ( postgresOnce sync.Once _globalPostgresClient *gorm.DB - _globalPostgresMu sync.RWMutex ) // GetPostgresDBClient returns the global PostgresDB client.It's safe for concurrent use. func GetPostgresDBClient() *gorm.DB { - _globalPostgresMu.RLock() - client := _globalPostgresClient - _globalPostgresMu.RUnlock() - return client + return _globalPostgresClient } // InitPostgresDBInstance return instance of PostgresDB client @@ -36,11 +32,19 @@ func InitPostgresDBInstance(ctx context.Context, PostgresDBURI string) *gorm.DB // initPostgresDBClient return successfully initialized PostgresDB client func initPostgresDBClient(ctx context.Context, PostgresDBURI string) *gorm.DB { - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() db, err := gorm.Open(postgres.Open(PostgresDBURI), &gorm.Config{Logger: logger.NewGormLogger()}) if err != nil { panic(err) } + + // Auto migrate async task tables + err = db.WithContext(ctx).AutoMigrate( + &orm.AsyncTask{}, + &orm.AsyncTaskResult{}, + ) + if err != nil { + panic(err) + } + return db } diff --git a/database/query_bay.go b/database/query_bay.go new file mode 100644 index 0000000..04ca639 --- /dev/null +++ b/database/query_bay.go @@ -0,0 +1,56 @@ +// Package database define database operation functions +package database + +import ( + "context" + "time" + + "modelRT/logger" + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// QueryBayByUUID returns the Bay record matching bayUUID. +func QueryBayByUUID(ctx context.Context, tx *gorm.DB, bayUUID uuid.UUID) (*orm.Bay, error) { + var bay orm.Bay + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("bay_uuid = ?", bayUUID). + Clauses(clause.Locking{Strength: "UPDATE"}). + First(&bay) + + if result.Error != nil { + return nil, result.Error + } + return &bay, nil +} + +// QueryBaysByUUIDs returns Bay records matching the given UUIDs in a single query. +// The returned slice preserves database order; unmatched UUIDs are silently omitted. +func QueryBaysByUUIDs(ctx context.Context, tx *gorm.DB, bayUUIDs []uuid.UUID) ([]orm.Bay, error) { + if len(bayUUIDs) == 0 { + return nil, nil + } + + var bays []orm.Bay + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Where("bay_uuid IN ?", bayUUIDs). + Clauses(clause.Locking{Strength: "UPDATE"}). + Find(&bays) + + if result.Error != nil { + logger.Error(ctx, "query bays by uuids failed", "error", result.Error) + return nil, result.Error + } + return bays, nil +} diff --git a/database/query_component.go b/database/query_component.go index 9e8798d..73ca27c 100644 --- a/database/query_component.go +++ b/database/query_component.go @@ -148,6 +148,39 @@ func QueryLongIdentModelInfoByToken(ctx context.Context, tx *gorm.DB, measTag st return &resultComp, &meauserment, nil } +// QueryComponentsInServiceByUUIDs returns a map of global_uuid → in_service for the +// given UUIDs. Only global_uuid and in_service columns are selected for efficiency. +func QueryComponentsInServiceByUUIDs(ctx context.Context, tx *gorm.DB, uuids []uuid.UUID) (map[uuid.UUID]bool, error) { + if len(uuids) == 0 { + return make(map[uuid.UUID]bool), nil + } + + type row struct { + GlobalUUID uuid.UUID `gorm:"column:global_uuid"` + InService bool `gorm:"column:in_service"` + } + + var rows []row + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Model(&orm.Component{}). + Select("global_uuid, in_service"). + Where("global_uuid IN ?", uuids). + Scan(&rows) + if result.Error != nil { + return nil, result.Error + } + + m := make(map[uuid.UUID]bool, len(rows)) + for _, r := range rows { + m[r.GlobalUUID] = r.InService + } + return m, nil +} + // QueryShortIdentModelInfoByToken define func to query short identity model info by short token func QueryShortIdentModelInfoByToken(ctx context.Context, tx *gorm.DB, measTag string, condition *orm.Component) (*orm.Component, *orm.Measurement, error) { var resultComp orm.Component diff --git a/database/query_component_measurement.go b/database/query_component_measurement.go index b8ddb7d..e9e95ca 100644 --- a/database/query_component_measurement.go +++ b/database/query_component_measurement.go @@ -2,8 +2,12 @@ package database import ( + "context" + "fmt" + "modelRT/orm" + "golang.org/x/sync/errgroup" "gorm.io/gorm" ) @@ -17,7 +21,7 @@ type StationWithParent struct { ZoneTag string `gorm:"column:zone_tag"` } -func GetFullMeasurementSet(db *gorm.DB) (*orm.MeasurementSet, error) { +func GetFullMeasurementSet(ctx context.Context, db *gorm.DB) (*orm.MeasurementSet, error) { mSet := &orm.MeasurementSet{ GridToZoneTags: make(map[string][]string), ZoneToStationTags: make(map[string][]string), @@ -26,85 +30,110 @@ func GetFullMeasurementSet(db *gorm.DB) (*orm.MeasurementSet, error) { CompTagToMeasTags: make(map[string][]string), } - var grids []orm.Grid - if err := db.Table("grid").Select("tagname").Scan(&grids).Error; err == nil { - for _, g := range grids { - if g.TAGNAME != "" { - mSet.AllGridTags = append(mSet.AllGridTags, g.TAGNAME) + g, gctx := errgroup.WithContext(ctx) + db = db.WithContext(gctx) + + g.Go(func() error { + var grids []orm.Grid + if err := db.Table("grid").Select("tagname").Scan(&grids).Error; err != nil { + return fmt.Errorf("query grids: %w", err) + } + for _, grid := range grids { + if grid.TAGNAME != "" { + mSet.AllGridTags = append(mSet.AllGridTags, grid.TAGNAME) } } - } + return nil + }) - var zones []struct { - orm.Zone - GridTag string `gorm:"column:grid_tag"` - } - if err := db.Table("zone"). - Select("zone.*, grid.tagname as grid_tag"). - Joins("left join grid on zone.grid_id = grid.id"). - Scan(&zones).Error; err == nil { + g.Go(func() error { + var zones []struct { + orm.Zone + GridTag string `gorm:"column:grid_tag"` + } + if err := db.Table("zone"). + Select("zone.*, grid.tagname as grid_tag"). + Joins("left join grid on zone.grid_id = grid.id"). + Scan(&zones).Error; err != nil { + return fmt.Errorf("query zones: %w", err) + } for _, z := range zones { mSet.AllZoneTags = append(mSet.AllZoneTags, z.TAGNAME) if z.GridTag != "" { mSet.GridToZoneTags[z.GridTag] = append(mSet.GridToZoneTags[z.GridTag], z.TAGNAME) } } - } + return nil + }) - var stations []struct { - orm.Station - ZoneTag string `gorm:"column:zone_tag"` - } - if err := db.Table("station"). - Select("station.*, zone.tagname as zone_tag"). - Joins("left join zone on station.zone_id = zone.id"). - Scan(&stations).Error; err == nil { + g.Go(func() error { + var stations []struct { + orm.Station + ZoneTag string `gorm:"column:zone_tag"` + } + if err := db.Table("station"). + Select("station.*, zone.tagname as zone_tag"). + Joins("left join zone on station.zone_id = zone.id"). + Scan(&stations).Error; err != nil { + return fmt.Errorf("query stations: %w", err) + } for _, s := range stations { mSet.AllStationTags = append(mSet.AllStationTags, s.TAGNAME) if s.ZoneTag != "" { mSet.ZoneToStationTags[s.ZoneTag] = append(mSet.ZoneToStationTags[s.ZoneTag], s.TAGNAME) } } - } + return nil + }) - var comps []struct { - orm.Component - StationTag string `gorm:"column:station_tag"` - } - if err := db.Table("component"). - Select("component.*, station.tagname as station_tag"). - Joins("left join station on component.station_id = station.id"). - Scan(&comps).Error; err == nil { + g.Go(func() error { + var comps []struct { + orm.Component + StationTag string `gorm:"column:station_tag"` + } + if err := db.Table("component"). + Select("component.*, station.tagname as station_tag"). + Joins("left join station on component.station_id = station.id"). + Scan(&comps).Error; err != nil { + return fmt.Errorf("query components: %w", err) + } for _, c := range comps { mSet.AllCompNSPaths = append(mSet.AllCompNSPaths, c.NSPath) mSet.AllCompTags = append(mSet.AllCompTags, c.Tag) - if c.StationTag != "" { mSet.StationToCompNSPaths[c.StationTag] = append(mSet.StationToCompNSPaths[c.StationTag], c.NSPath) } - if c.NSPath != "" { mSet.CompNSPathToCompTags[c.NSPath] = append(mSet.CompNSPathToCompTags[c.NSPath], c.Tag) } } - } + return nil + }) - mSet.AllConfigTags = append(mSet.AllConfigTags, "bay") - - var measurements []struct { - orm.Measurement - CompTag string `gorm:"column:comp_tag"` - } - if err := db.Table("measurement"). - Select("measurement.*, component.tag as comp_tag"). - Joins("left join component on measurement.component_uuid = component.global_uuid"). - Scan(&measurements).Error; err == nil { + g.Go(func() error { + var measurements []struct { + orm.Measurement + CompTag string `gorm:"column:comp_tag"` + } + if err := db.Table("measurement"). + Select("measurement.*, component.tag as comp_tag"). + Joins("left join component on measurement.component_uuid = component.global_uuid"). + Scan(&measurements).Error; err != nil { + return fmt.Errorf("query measurements: %w", err) + } for _, m := range measurements { mSet.AllMeasTags = append(mSet.AllMeasTags, m.Tag) if m.CompTag != "" { mSet.CompTagToMeasTags[m.CompTag] = append(mSet.CompTagToMeasTags[m.CompTag], m.Tag) } } + return nil + }) + + if err := g.Wait(); err != nil { + return nil, err } + + mSet.AllConfigTags = append(mSet.AllConfigTags, "bay") return mSet, nil } diff --git a/database/query_topologic.go b/database/query_topologic.go index eea21b1..4799875 100644 --- a/database/query_topologic.go +++ b/database/query_topologic.go @@ -32,71 +32,51 @@ func QueryTopologic(ctx context.Context, tx *gorm.DB) ([]orm.Topologic, error) { return topologics, nil } -// QueryTopologicFromDB return the result of query topologic info from DB -func QueryTopologicFromDB(ctx context.Context, tx *gorm.DB) (*diagram.MultiBranchTreeNode, error) { +// QueryTopologicByStartUUID returns all edges reachable from startUUID following +// directed uuid_from → uuid_to edges in the topologic table. +func QueryTopologicByStartUUID(ctx context.Context, tx *gorm.DB, startUUID uuid.UUID) ([]orm.Topologic, error) { + var topologics []orm.Topologic + + cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + result := tx.WithContext(cancelCtx). + Clauses(clause.Locking{Strength: "UPDATE"}). + Raw(sql.RecursiveSQL, startUUID). + Scan(&topologics) + if result.Error != nil { + logger.Error(ctx, "query topologic by start uuid failed", "start_uuid", startUUID, "error", result.Error) + return nil, result.Error + } + return topologics, nil +} + +// QueryTopologicFromDB return the result of query topologic info from DB. +// Returns the root node and a flat nodeMap for O(1) lookup by UUID. +func QueryTopologicFromDB(ctx context.Context, tx *gorm.DB) (*diagram.MultiBranchTreeNode, map[uuid.UUID]*diagram.MultiBranchTreeNode, error) { topologicInfos, err := QueryTopologic(ctx, tx) if err != nil { logger.Error(ctx, "query topologic info failed", "error", err) - return nil, err + return nil, nil, err } - tree, err := BuildMultiBranchTree(topologicInfos) + tree, nodeMap, err := BuildMultiBranchTree(topologicInfos) if err != nil { logger.Error(ctx, "init topologic failed", "error", err) - return nil, err + return nil, nil, err } - return tree, nil + return tree, nodeMap, nil } -// InitCircuitDiagramTopologic return circuit diagram topologic info from postgres -func InitCircuitDiagramTopologic(topologicNodes []orm.Topologic) error { - var rootVertex *diagram.MultiBranchTreeNode - for _, node := range topologicNodes { - if node.UUIDFrom == constants.UUIDNil { - rootVertex = diagram.NewMultiBranchTree(node.UUIDFrom) - break - } - } - - if rootVertex == nil { - return fmt.Errorf("root vertex is nil") - } - - for _, node := range topologicNodes { - if node.UUIDFrom == constants.UUIDNil { - nodeVertex := diagram.NewMultiBranchTree(node.UUIDTo) - rootVertex.AddChild(nodeVertex) - } - } - - node := rootVertex - for _, nodeVertex := range node.Children { - nextVertexs := make([]*diagram.MultiBranchTreeNode, 0) - nextVertexs = append(nextVertexs, nodeVertex) - } - return nil -} - -// TODO 电流互感器不单独划分间隔,以母线、浇筑母线、变压器为间隔原件 -func IntervalBoundaryDetermine(uuid uuid.UUID) bool { - diagram.GetComponentMap(uuid.String()) - // TODO 判断 component 的类型是否为间隔 - // TODO 0xA1B2C3D4,高四位表示可以成为间隔的compoent类型的值为FFFF,普通 component 类型的值为 0000。低四位中前二位表示component的一级类型，例如母线 PT、母联/母分、进线等，低四位中后二位表示一级类型中包含的具体类型，例如母线 PT中包含的电压互感器、隔离开关、接地开关、避雷器、带电显示器等。 - num := uint32(0xA1B2C3D4) // 八位16进制数 - high16 := uint16(num >> 16) - fmt.Printf("原始值: 0x%X\n", num) // 输出: 0xA1B2C3D4 - fmt.Printf("高十六位: 0x%X\n", high16) // 输出: 0xA1B2 - return true -} - -// BuildMultiBranchTree return the multi branch tree by topologic info and component type map -func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeNode, error) { +// BuildMultiBranchTree return the multi branch tree by topologic info. +// Returns the root node and a flat nodeMap for O(1) lookup by UUID. +func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeNode, map[uuid.UUID]*diagram.MultiBranchTreeNode, error) { nodeMap := make(map[uuid.UUID]*diagram.MultiBranchTreeNode, len(topologics)*2) for _, topo := range topologics { if _, exists := nodeMap[topo.UUIDFrom]; !exists { - // skip special uuid - if topo.UUIDTo != constants.UUIDNil { + // UUIDNil is the virtual root sentinel — skip creating a regular node for it + if topo.UUIDFrom != constants.UUIDNil { nodeMap[topo.UUIDFrom] = &diagram.MultiBranchTreeNode{ ID: topo.UUIDFrom, Children: make([]*diagram.MultiBranchTreeNode, 0), @@ -105,7 +85,6 @@ func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeN } if _, exists := nodeMap[topo.UUIDTo]; !exists { - // skip special uuid if topo.UUIDTo != constants.UUIDNil { nodeMap[topo.UUIDTo] = &diagram.MultiBranchTreeNode{ ID: topo.UUIDTo, @@ -118,10 +97,13 @@ func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeN for _, topo := range topologics { var parent *diagram.MultiBranchTreeNode if topo.UUIDFrom == constants.UUIDNil { - parent = &diagram.MultiBranchTreeNode{ - ID: constants.UUIDNil, + if _, exists := nodeMap[constants.UUIDNil]; !exists { + nodeMap[constants.UUIDNil] = &diagram.MultiBranchTreeNode{ + ID: constants.UUIDNil, + Children: make([]*diagram.MultiBranchTreeNode, 0), + } } - nodeMap[constants.UUIDNil] = parent + parent = nodeMap[constants.UUIDNil] } else { parent = nodeMap[topo.UUIDFrom] } @@ -141,7 +123,7 @@ func BuildMultiBranchTree(topologics []orm.Topologic) (*diagram.MultiBranchTreeN // return root vertex root, exists := nodeMap[constants.UUIDNil] if !exists { - return nil, fmt.Errorf("root node not found") + return nil, nil, fmt.Errorf("root node not found") } - return root, nil + return root, nodeMap, nil } diff --git a/database/update_component.go b/database/update_component.go index 7957bc8..e08eea9 100644 --- a/database/update_component.go +++ b/database/update_component.go @@ -43,7 +43,7 @@ func UpdateComponentIntoDB(ctx context.Context, tx *gorm.DB, componentInfo netwo Name: componentInfo.Name, Context: componentInfo.Context, Op: componentInfo.Op, - Ts: time.Now(), + TS: time.Now(), } result = tx.Model(&orm.Component{}).WithContext(cancelCtx).Where("GLOBAL_UUID = ?", component.GlobalUUID).Updates(&updateParams) diff --git a/deploy/deploy.md b/deploy/deploy.md index 3364838..7b758b0 100644 --- a/deploy/deploy.md +++ b/deploy/deploy.md @@ -1,12 +1,12 @@ -# 项目依赖服务部署指南 +# 项目服务部署指南 -本项目依赖于 $\text{PostgreSQL}$ 数据库和 $\text{Redis Stack Server}$（包含 $\text{Redisearch}$ 等模块）部署文档将使用 $\text{Docker}$ 容器化技术部署这两个依赖服务 +本项目依赖于 `PostgreSQL` 数据库和 `Redis Stack Server`（包含 `Redisearch` 等模块）部署文档将使用 `Docker` 容器化技术部署这两个依赖服务 ## 前提条件 -1. 已安装 $\text{Docker}$ +1. 已安装 `Docker` 2. 下载相关容器镜像 -3. 确保主机的 $\text{5432}$ 端口（$\text{Postgres}$）和 $\text{6379}$ 端口（$\text{Redis}$）未被占用 +3. 确保主机的 `5432` 端口（`Postgres`）和 `6379` 端口（`Redis`）未被占用 ### 1\. 部署 PostgreSQL 数据库 @@ -14,7 +14,7 @@ #### 1.1 部署命令 -运行以下命令启动 $\text{PostgreSQL}$ 容器 +运行以下命令启动 `PostgreSQL` 容器 ```bash docker run --name postgres \ @@ -45,13 +45,75 @@ docker ps -a ｜grep postgres docker logs postgres ``` +#### 1.4 初始化异步任务表 + +`PostgreSQL` 启动后执行以下建表语句，创建异步任务系统所需的两张表： + +```sql +-- ========================================== +-- 表: async_task +-- 说明: 存储异步任务的生命周期跟踪信息 +-- ========================================== +CREATE TABLE IF NOT EXISTS async_task ( + task_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + task_type VARCHAR(50) NOT NULL, + status VARCHAR(20) NOT NULL, + params JSONB, + created_at BIGINT NOT NULL, + finished_at BIGINT, + started_at BIGINT, + execution_time BIGINT, + progress INTEGER, + retry_count INTEGER DEFAULT 0, + max_retry_count INTEGER DEFAULT 3, + next_retry_time BIGINT, + retry_delay INTEGER DEFAULT 5000, + priority INTEGER DEFAULT 5, + queue_name VARCHAR(100) DEFAULT 'default', + worker_id VARCHAR(50), + failure_reason TEXT, + stack_trace TEXT, + created_by VARCHAR(100) +); + +CREATE INDEX IF NOT EXISTS idx_async_task_task_type ON async_task(task_type); +CREATE INDEX IF NOT EXISTS idx_async_task_status ON async_task(status); +CREATE INDEX IF NOT EXISTS idx_async_task_created_at ON async_task(created_at); +CREATE INDEX IF NOT EXISTS idx_async_task_finished_at ON async_task(finished_at); +CREATE INDEX IF NOT EXISTS idx_async_task_started_at ON async_task(started_at); +CREATE INDEX IF NOT EXISTS idx_async_task_next_retry_time ON async_task(next_retry_time); +CREATE INDEX IF NOT EXISTS idx_async_task_priority ON async_task(priority); +CREATE INDEX IF NOT EXISTS idx_async_task_status_retry ON async_task(status, next_retry_time) + WHERE status = 'FAILED' AND next_retry_time IS NOT NULL; + +-- ========================================== +-- 表: async_task_result +-- 说明: 存储异步任务的执行结果 +-- ========================================== +CREATE TABLE IF NOT EXISTS async_task_result ( + task_id UUID PRIMARY KEY, + result JSONB, + error_code INTEGER, + error_message TEXT, + error_detail JSONB, + execution_time BIGINT NOT NULL DEFAULT 0, + memory_usage BIGINT, + cpu_usage DOUBLE PRECISION, + retry_count INTEGER DEFAULT 0, + completed_at BIGINT NOT NULL +); + +COMMENT ON TABLE async_task IS '异步任务生命周期跟踪表'; +COMMENT ON TABLE async_task_result IS '异步任务执行结果表'; +``` + ### 2\. 部署 Redis Stack Server -我们将使用 `redis/redis-stack-server:latest` 镜像该镜像内置了 $\text{Redisearch}$ 模块，用于 $\text{ModelRT}$ 项目中补全功能 +我们将使用 `redis/redis-stack-server:latest` 镜像该镜像内置了 `Redisearch` 模块，用于 `ModelRT` 项目中补全功能 #### 2.1 部署命令 -运行以下命令启动 $\text{Redis Stack Server}$ 容器 +运行以下命令启动 `Redis Stack Server` 容器 ```bash docker run --name redis -p 6379:6379 \ @@ -68,7 +130,7 @@ docker run --name redis -p 6379:6379 \ | **地址** | `localhost:6379` | | | **密码** | **无** | 默认未设置密码 | -> **注意:** 生产环境中建议使用 `-e REDIS_PASSWORD=` 参数来设置 $\text{Redis}$ 访问密码 +> **注意:** 生产环境中建议使用 `-e REDIS_PASSWORD=` 参数来设置 `Redis` 访问密码 #### 2.3 状态检查 @@ -136,7 +198,7 @@ VALUES 'ns1', 'tag1', 'component1', 'bus_1', '', 'grid1', 'zone1', 'station1', 1, -1, - false, + true, -1, -1, '{}', '{}', @@ -149,7 +211,7 @@ VALUES 'ns2', 'tag2', 'component2', 'bus_1', '', 'grid1', 'zone1', 'station1', 1, -1, - false, + true, -1, -1, '{}', '{}', @@ -162,13 +224,78 @@ VALUES 'ns3', 'tag3', 'component3', 'bus_1', '', 'grid1', 'zone1', 'station2', 2, -1, - false, + true, -1, -1, '{}', '{}', '{}', -1, CURRENT_TIMESTAMP +), +( + '70c190f2-8a60-42a9-b143-ec5f87e0aa6b', + 'ns4', 'tag4', 'component4', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + '10f155cf-bd27-4557-85b2-d126b6e2657f', + 'ns5', 'tag5', 'component5', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + 'e32bc0be-67f4-4d79-a5da-eaa40a5bd77d', + 'ns6', 'tag6', 'component6', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + '70c190f2-8a75-42a9-b166-ec5f87e0aa6b', + 'ns7', 'tag7', 'component7', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP +), +( + '70c200f2-8a75-42a9-c166-bf5f87e0aa6b', + 'ns8', 'tag8', 'component8', 'bus_1', '', + 'grid1', 'zone1', 'station1', 1, + -1, + true, + -1, -1, + '{}', + '{}', + '{}', + -1, + CURRENT_TIMESTAMP ); INSERT INTO public.measurement (id, tag, name, type, size, data_source, event_plan, bay_uuid, component_uuid, op, ts) @@ -276,46 +403,46 @@ go run deploy/redis-test-data/measurments-recommend/measurement_injection.go | 类别 | 参数名 | 作用描述 | 示例值 | | :--- | :--- | :--- | :--- | -| **Postgres** | `host` | PostgreSQL 数据库服务器的 $\text{IP}$ 地址或域名。 | `"192.168.1.101"` | +| **Postgres** | `host` | PostgreSQL 数据库服务器的 `IP` 地址或域名。 | `"192.168.1.101"` | | | `port` | PostgreSQL 数据库服务器的端口号。 | `5432` | | | `database` | 连接的数据库名称。 | `"demo"` | | | `user` | 连接数据库所使用的用户名。 | `"postgres"` | | | `password` | 连接数据库所使用的密码。 | `"coslight"` | -| **Kafka** | `servers` | Kafka 集群的 $\text{Bootstrap Server}$ 地址列表（通常是 $\text{host:port}$ 形式，多个地址用逗号分隔）。 | `"localhost:9092"` | +| **Kafka** | `servers` | Kafka 集群的 `Bootstrap Server` 地址列表（通常是 `host:port` 形式，多个地址用逗号分隔）。 | `"localhost:9092"` | | | `port` | Kafka 服务器的端口号。 | `9092` | -| | `group_id` | 消费者组 $\text{ID}$，用于标识和管理一组相关的消费者。 | `"modelRT"` | +| | `group_id` | 消费者组 `ID`，用于标识和管理一组相关的消费者。 | `"modelRT"` | | | `topic` | Kafka 消息的主题名称。 | `""` | -| | `auto_offset_reset` | 消费者首次启动或 $\text{Offset}$ 无效时，从哪个位置开始消费（如 `earliest` 或 `latest`）。 | `"earliest"` | -| | `enable_auto_commit` | 是否自动提交 $\text{Offset}$。设为 $\text{false}$ 通常用于手动控制 $\text{Offset}$ 提交。 | `"false"` | +| | `auto_offset_reset` | 消费者首次启动或 `Offset` 无效时，从哪个位置开始消费（如 `earliest` 或 `latest`）。 | `"earliest"` | +| | `enable_auto_commit` | 是否自动提交 `Offset`。设为 `false` 通常用于手动控制 `Offset` 提交。 | `"false"` | | | `read_message_time_duration` | 读取消息时的超时或等待时间。 | `”0.5s"` | | **Logger (Zap)** | `mode` | 日志模式，通常为 `development`（开发）或 `production`（生产）。影响日志格式。 | `"development"` | -| | `level` | 最低日志级别（如 $\text{debug, info, warn, error}$）。 | `"debug"` | +| | `level` | 最低日志级别（如 `debug`, `info`, `warn`, `error`）。 | `"debug"` | | | `filepath` | 日志文件的输出路径和名称格式（`%s` 会被替换为日期等）。 | `"/Users/douxu/Workspace/coslight/modelRT/modelRT-%s.log"` | -| | `maxsize` | 单个日志文件最大大小（单位：$\text{MB}$）。 | `1` | +| | `maxsize` | 单个日志文件最大大小（单位：`MB`）。 | `1` | | | `maxbackups` | 保留旧日志文件的最大个数。 | `5` | | | `maxage` | 保留旧日志文件的最大天数。 | `30` | | | `compress` | 是否压缩备份的日志文件。 | `false` | | **Ants Pool** | `parse_concurrent_quantity` | 用于解析任务的协程池最大并发数量。 | `10` | | | `rtd_receive_concurrent_quantity` | 用于实时数据接收任务的协程池最大并发数量。 | `10` | -| **Locker Redis** | `addr` | 分布式锁服务所使用的 $\text{Redis}$ 地址。 | `"127.0.0.1:6379"` | -| | `password` | $\text{Locker Redis}$ 的密码。 | `""` | -| | `db` | $\text{Locker Redis}$ 使用的数据库编号。 | `1` | -| | `poolsize` | $\text{Locker Redis}$ 连接池的最大连接数。 | `50` | -| | `timeout` | $\text{Locker Redis}$ 连接操作的超时时间（单位：毫秒）。 | `10` | -| **Storage Redis** | `addr` | 数据存储服务所使用的 $\text{Redis}$ 地址（例如 $\text{Redisearch}$）。 | `"127.0.0.1:6379"` | -| | `password` | $\text{Storage Redis}$ 的密码。 | `""` | -| | `db` | $\text{Storage Redis}$ 使用的数据库编号。 | `0` | -| | `poolsize` | $\text{Storage Redis}$ 连接池的最大连接数。 | `50` | -| | `timeout` | $\text{Storage Redis}$ 连接操作的超时时间（单位：毫秒）。 | `10` | -| **Base Config** | `grid_id` | 项目所操作的默认电网 $\text{ID}$。 | `1` | -| | `zone_id` | 项目所操作的默认区域 $\text{ID}$。 | `1` | -| | `station_id` | 项目所操作的默认变电站 $\text{ID}$。 | `1` | +| **Locker Redis** | `addr` | 分布式锁服务所使用的 `Redis` 地址。 | `"127.0.0.1:6379"` | +| | `password` | `Locker Redis` 的密码。 | `""` | +| | `db` | `Locker Redis` 使用的数据库编号。 | `1` | +| | `poolsize` | `Locker Redis` 连接池的最大连接数。 | `50` | +| | `timeout` | `Locker Redis` 连接操作的超时时间（单位：毫秒）。 | `10` | +| **Storage Redis** | `addr` | 数据存储服务所使用的 `Redis` 地址（例如 `Redisearch`）。 | `"127.0.0.1:6379"` | +| | `password` | `Storage Redis` 的密码。 | `""` | +| | `db` | `Storage Redis` 使用的数据库编号。 | `0` | +| | `poolsize` | `Storage Redis` 连接池的最大连接数。 | `50` | +| | `timeout` | `Storage Redis` 连接操作的超时时间（单位：毫秒）。 | `10` | +| **Base Config** | `grid_id` | 项目所操作的默认电网 `ID`。 | `1` | +| | `zone_id` | 项目所操作的默认区域 `ID`。 | `1` | +| | `station_id` | 项目所操作的默认变电站 `ID`。 | `1` | | **Service Config** | `service_name` | 服务名称，用于日志、监控等标识。 | `"modelRT"` | | | `secret_key` | 服务内部使用的秘钥，用于签名或认证。 | `"modelrt_key"` | -| **DataRT API** | `host` | 外部 $\text{DataRT}$ 服务的主机地址。 | `"http://127.0.0.1"` | -| | `port` | $\text{DataRT}$ 服务的端口号。 | `8888` | -| | `polling_api` | 轮询数据的 $\text{API}$ 路径。 | `"datart/getPointData"` | -| | `polling_api_method` | 调用该 $\text{API}$ 使用的 $\text{HTTP}$ 方法。 | `"GET"` | +| **DataRT API** | `host` | 外部 `DataRT` 服务的主机地址。 | `"http://127.0.0.1"` | +| | `port` | `DataRT` 服务的端口号。 | `8888` | +| | `polling_api` | 轮询数据的 `API` 路径。 | `"datart/getPointData"` | +| | `polling_api_method` | 调用该 `API` 使用的 `HTTP` 方法。 | `"GET"` | #### 3.2 编译 ModelRT 服务 @@ -336,15 +463,561 @@ go build -o model-rt main.go 在发现控制台输出如下信息`starting ModelRT server` 后即代表服务启动成功 -### 4\. 后续操作（停止与清理） +### 4\. 部署基础依赖（Kubernetes） -#### 4.1 停止容器 +Redis 和 RabbitMQ 部署在 Minikube 中，YAML 文件位于 `deploy/k8s/`。RabbitMQ 启用双向 TLS（mTLS），客户端以 X.509 证书的 CN 字段作为用户名进行认证。 + +#### 4.1 部署 Redis + +```bash +kubectl apply -f deploy/k8s/redis-deployment.yaml +kubectl apply -f deploy/k8s/redis-service.yaml +``` + +| 参数 | 值 | 说明 | +| :--- | :--- | :--- | +| **镜像** | `redis/redis-stack-server:latest` | 内置 Redisearch 模块 | +| **NodePort** | `30001` | 集群外访问端口 | + +#### 4.2 RabbitMQ TLS 证书生成 + +RabbitMQ 配置为仅允许 TLS 连接（`listeners.tcp = none`），所有客户端须持有由同一 CA 签发的证书。 + +##### 4.2.1 生成根 CA + +```bash +# 克隆 tls-gen 工具 +git clone https://github.com/rabbitmq/tls-gen.git +cd tls-gen/basic + +# 生成根 CA（结果在 result/ 目录） +make CN=rabbitmq-server +# ca_certificate.pem 和 ca_key.pem 生成于 result/ +``` + +##### 4.2.2 生成服务器证书 + +服务器证书需包含 SAN（Subject Alternative Name），使其同时匹配集群内 DNS 和 Minikube IP。 + +创建 `server.cnf`： + +```text +[req] +distinguished_name = req_distinguished_name +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing +O = coslight +CN = rabbitmq-server + +[v3_server] +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth, clientAuth +subjectAltName = @alt_names + +[alt_names] +DNS.1 = rabbitmq-server +DNS.2 = rabbitmq-service.default.svc.cluster.local +DNS.3 = localhost +IP.1 = 192.168.49.2 +IP.2 = 127.0.0.1 +``` + +生成证书： + +```bash +# 将 ca_certificate.pem 和 ca_key.pem（即 cakey.pem）放在当前目录 +openssl genrsa -out server_key.pem 2048 + +openssl req -new -key server_key.pem -out server_cert.csr -config server.cnf + +openssl x509 -req -in server_cert.csr \ + -CA ca_certificate.pem -CAkey cakey.pem -CAcreateserial \ + -out server_certificate.pem -days 730 -sha256 \ + -extfile server.cnf -extensions v3_server + +rm server_cert.csr +``` + +##### 4.2.3 生成 ModelRT 客户端证书 + +CN 必须与 RabbitMQ 中注册的用户名一致（`modelrt-client`）。 + +创建 `modelrt.cnf`： + +```text +[req] +distinguished_name = req_distinguished_name +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing +O = coslight +CN = modelrt-client + +[v3_client] +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +``` + +生成证书： + +```bash +openssl genrsa -out modelrt_client_key.pem 2048 + +openssl req -new -key modelrt_client_key.pem \ + -out modelrt_client.csr -config modelrt.cnf + +openssl x509 -req -in modelrt_client.csr \ + -CA ca_certificate.pem -CAkey cakey.pem -CAcreateserial \ + -out modelrt_client_cert.pem -days 365 \ + -extensions v3_client -extfile modelrt.cnf + +rm modelrt_client.csr +``` + +##### 4.2.4 生成 EventRT 客户端证书 + +创建 `eventrt.cnf`（CN 改为 `eventrt-client`）： + +```text +[req] +distinguished_name = req_distinguished_name +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing +O = coslight +CN = eventrt-client + +[v3_client] +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +``` + +生成证书： + +```bash +openssl genrsa -out eventrt_client_key.pem 2048 + +openssl req -new -key eventrt_client_key.pem \ + -out eventrt_client.csr -config eventrt.cnf + +openssl x509 -req -in eventrt_client.csr \ + -CA ca_certificate.pem -CAkey cakey.pem -CAcreateserial \ + -out eventrt_client_cert.pem -days 365 \ + -extensions v3_client -extfile eventrt.cnf + +rm eventrt_client.csr +``` + +##### 4.2.5 验证证书 + +```bash +# 验证服务器证书 +openssl verify -CAfile ca_certificate.pem server_certificate.pem + +# 验证客户端证书 +openssl verify -CAfile ca_certificate.pem modelrt_client_cert.pem +openssl verify -CAfile ca_certificate.pem eventrt_client_cert.pem + +# 查看证书详情（确认 CN 和 SAN） +openssl x509 -in server_certificate.pem -noout -subject -ext subjectAltName +openssl x509 -in modelrt_client_cert.pem -noout -subject +openssl x509 -in eventrt_client_cert.pem -noout -subject +``` + +#### 4.3 部署 RabbitMQ + +##### 4.3.1 创建证书 Secret + +将服务器端三个证书文件打包为 K8s Secret（在证书文件所在目录执行）： + +```bash +kubectl create secret generic rabbitmq-certs \ + --from-file=ca_certificate.pem=./ca_certificate.pem \ + --from-file=server_certificate.pem=./server_certificate.pem \ + --from-file=server_key.pem=./server_key.pem +``` + +##### 4.3.2 部署 + +```bash +kubectl apply -f deploy/k8s/rabbitmq-secret.yaml +kubectl apply -f deploy/k8s/rabbitmq-config.yaml +kubectl apply -f deploy/k8s/rabbitmq-users-config.yaml +kubectl apply -f deploy/k8s/rabbitmq-deployment.yaml +kubectl apply -f deploy/k8s/rabbitmq-service.yaml +``` + +##### 4.3.3 端口汇总 + +| 端口 | NodePort | 说明 | +| :--- | :--- | :--- | +| `5671` | `30671` | AMQP over TLS（客户端连接） | +| `5672` | `30672` | AMQP 明文（内部备用，生产禁用） | +| `15671` | `31671` | Management UI over TLS | +| `15672` | `31672` | Management UI 明文（内部备用） | + +##### 4.3.4 用户与权限说明 + +用户定义在 `rabbitmq-users-config.yaml` 的 `definitions.json` 中，通过 `load_definitions` 启动时自动加载： + +| 用户 | 认证方式 | 权限 | 说明 | +| :--- | :--- | :--- | :--- | +| `coslight` | 密码 | administrator | 管理员，密码在 rabbitmq-secret.yaml | +| `modelrt-client` | X.509 证书（CN） | configure/read/write | ModelRT 服务专用 | +| `eventrt-client` | X.509 证书（CN） | configure/read/write | EventRT 服务专用 | +| `web-client` | X.509 证书（CN） | read/write | Web 客户端 | + +> **注意：** 证书认证用户的 `password_hash` 留空；RabbitMQ 通过 `ssl_cert_login_from = common_name` 将证书 CN 映射为用户名。 + +#### 4.4 部署 PostgreSQL + +```bash +kubectl apply -f deploy/k8s/pg-configmap.yaml +kubectl apply -f deploy/k8s/pg-pvc.yaml +kubectl apply -f deploy/k8s/pg-statefulset.yaml +kubectl apply -f deploy/k8s/pg-service.yaml +``` + +| 参数 | 值 | 说明 | +| :--- | :--- | :--- | +| **镜像** | `postgres:13.16` | PostgreSQL 13.16 | +| **NodePort** | `30432` | 集群外访问端口 | +| **数据库** | `demo` | ConfigMap 中 `POSTGRES_DB` | +| **用户名** | `postgres` | ConfigMap 中 `POSTGRES_USER` | +| **密码** | `coslight` | ConfigMap `postgres-config` 中配置，生产环境迁移至 Secret | +| **存储** | `2Gi` | PVC `postgres-data` | + +##### 4.4.1 等待 Pod 就绪 + +```bash +kubectl wait --for=condition=ready pod -l app=postgres --timeout=120s +``` + +##### 4.4.2 初始化异步任务表 + +PostgreSQL 就绪后执行 1.4 节的建表 SQL，可通过以下方式进入容器执行： + +```bash +# 交互式 psql +kubectl exec -it $(kubectl get pod -l app=postgres -o jsonpath='{.items[0].metadata.name}') \ + -- psql -U postgres -d demo + +# 或将 SQL 文件通过管道一次性执行 +kubectl exec -i $(kubectl get pod -l app=postgres -o jsonpath='{.items[0].metadata.name}') \ + -- psql -U postgres -d demo < /path/to/init.sql +``` + +##### 4.4.3 状态检查 + +```bash +kubectl get pods -l app=postgres +kubectl logs -l app=postgres --tail=30 +``` + +##### 4.4.4 清理 + +```bash +kubectl delete -f deploy/k8s/pg-service.yaml \ + -f deploy/k8s/pg-statefulset.yaml \ + -f deploy/k8s/pg-pvc.yaml \ + -f deploy/k8s/pg-configmap.yaml +``` + +#### 4.5 部署 MongoDB + +```bash +kubectl apply -f deploy/k8s/mongodb-secret.yaml +kubectl apply -f deploy/k8s/mongodb-pvc.yaml +kubectl apply -f deploy/k8s/mongodb-statefulset.yaml +kubectl apply -f deploy/k8s/mongodb-service.yaml +``` + +| 参数 | 值 | 说明 | +| :--- | :--- | :--- | +| **镜像** | `mongo:7.0` | MongoDB 7.0 | +| **NodePort** | `30017` | 集群外访问端口 | +| **用户名** | `admin` | Root 管理员 | +| **密码** | `coslight` | Secret `mongodb-secret` 中配置，生产环境请替换强密码 | +| **存储** | `2Gi` | PVC `mongodb-data` | + +> **注意：** 密码存储在 `mongodb-secret.yaml` 的 `stringData` 中，生产环境应替换为强密码，并避免将明文密码提交至版本库。 + +##### 4.5.1 等待 Pod 就绪 + +```bash +kubectl wait --for=condition=ready pod -l app=mongodb --timeout=120s +``` + +##### 4.5.2 连接验证 + +```bash +kubectl exec -it $(kubectl get pod -l app=mongodb -o jsonpath='{.items[0].metadata.name}') \ + -- mongosh -u admin -p coslight --authenticationDatabase admin +``` + +##### 4.5.3 状态检查 + +```bash +kubectl get pods -l app=mongodb +kubectl logs -l app=mongodb --tail=30 +``` + +##### 4.5.4 清理 + +```bash +kubectl delete -f deploy/k8s/mongodb-service.yaml \ + -f deploy/k8s/mongodb-statefulset.yaml \ + -f deploy/k8s/mongodb-pvc.yaml \ + -f deploy/k8s/mongodb-secret.yaml +``` + +### 5\. 部署 ModelRT（Kubernetes） + +所有资源部署在 `default` 命名空间，YAML 文件位于 `deploy/k8s/`。 + +#### 5.1 构建并推送镜像 + +```bash +# 在项目根目录执行 +docker build -f deploy/dockerfile/modelrt.Dockerfile -t coslight/modelrt:latest . + +# 推送到镜像仓库（或直接加载到 Minikube） +minikube image load coslight/modelrt:latest +``` + +#### 5.2 创建客户端证书 Secret + +在 RabbitMQ TLS 证书生成完成后（见 4.2），进入证书文件所在目录执行： + +```bash +sh deploy/k8s/modelrt-certs-secret.sh +``` + +该脚本等价于： + +```bash +kubectl create secret generic modelrt-certs \ + --from-file=ca_certificate.pem=./ca_certificate.pem \ + --from-file=modelrt_client_cert.pem=./modelrt_client_cert.pem \ + --from-file=modelrt_client_key.pem=./modelrt_client_key.pem +``` + +#### 5.3 部署 + +```bash +kubectl apply -f deploy/k8s/modelrt-secret.yaml +kubectl apply -f deploy/k8s/modelrt-configmap.yaml +kubectl apply -f deploy/k8s/modelrt-deployment.yaml +kubectl apply -f deploy/k8s/modelrt-service.yaml +``` + +#### 5.4 配置说明 + +| 配置项 | 方式 | 说明 | +| :--- | :--- | :--- | +| `postgres.password` | Secret `modelrt-secret` | 不写入 ConfigMap | +| `service.secret_key` | Secret `modelrt-secret` | 不写入 ConfigMap | +| RabbitMQ 客户端证书 | Secret `modelrt-certs` | 挂载至 `/app/configs/certs/` | +| `config.yaml` 其余配置 | ConfigMap `modelrt-config` | 所有 host 已替换为 K8s service 名 | +| `K8S_NAMESPACE` / `K8S_NODE_NAME` | Downward API | 注入至日志全局字段 | + +> **注意：** `modelrt-configmap.yaml` 中 `postgres.password` 和 `service.secret_key` 留空，实际值由容器启动时的环境变量 `POSTGRES_PASSWORD` / `SERVICE_SECRET_KEY` 注入，应用需读取这两个环境变量覆盖 config 中的空值。若应用当前仅读取文件配置，可直接将值填入 `modelrt-secret.yaml` 并在 ConfigMap 中引用，或在 ConfigMap 中直接填写。 + +#### 5.5 状态检查 + +```bash +# 查看 Pod 状态 +kubectl get pods -l app=modelrt + +# 查看启动日志 +kubectl logs -l app=modelrt --tail=50 + +# 查看 Service +kubectl get svc modelrt-service +``` + +#### 5.6 端口汇总 + +| NodePort | 说明 | +| :--- | :--- | +| `30080` | ModelRT HTTP API，SSH 隧道本地端口 `8080` | + +#### 5.7 清理 + +```bash +kubectl delete -f deploy/k8s/modelrt-service.yaml \ + -f deploy/k8s/modelrt-deployment.yaml \ + -f deploy/k8s/modelrt-configmap.yaml \ + -f deploy/k8s/modelrt-secret.yaml +kubectl delete secret modelrt-certs +``` + +### 6\. 部署可观测性栈（Kubernetes） + +在 `Kubernetes` 集群中部署 `Jaeger`（链路追踪）+ `Loki + Promtail + Grafana`（日志可视化）。所有资源部署在 `default` 命名空间，`YAML` 文件位于 `deploy/k8s/`。 + +#### 6.1 部署 Jaeger + +```bash +kubectl apply -f deploy/k8s/jaeger-deployment.yaml +kubectl apply -f deploy/k8s/jaeger-service.yaml +``` + +#### 6.2 部署 Loki + +```bash +kubectl apply -f deploy/k8s/loki-configmap.yaml +kubectl apply -f deploy/k8s/loki-pvc.yaml +kubectl apply -f deploy/k8s/loki-deployment.yaml +kubectl apply -f deploy/k8s/loki-service.yaml +``` + +#### 6.3 部署 Promtail + +```bash +kubectl apply -f deploy/k8s/promtail-rbac.yaml +kubectl apply -f deploy/k8s/promtail-configmap.yaml +kubectl apply -f deploy/k8s/promtail-daemonset.yaml +``` + +#### 6.4 部署 Grafana + +```bash +kubectl apply -f deploy/k8s/grafana-configmap.yaml +kubectl apply -f deploy/k8s/grafana-deployment.yaml +kubectl apply -f deploy/k8s/grafana-service.yaml +``` + +#### 6.5 一键部署 + +```bash +kubectl apply -f deploy/k8s/jaeger-deployment.yaml \ + -f deploy/k8s/jaeger-service.yaml \ + -f deploy/k8s/loki-configmap.yaml \ + -f deploy/k8s/loki-pvc.yaml \ + -f deploy/k8s/loki-deployment.yaml \ + -f deploy/k8s/loki-service.yaml \ + -f deploy/k8s/promtail-rbac.yaml \ + -f deploy/k8s/promtail-configmap.yaml \ + -f deploy/k8s/promtail-daemonset.yaml \ + -f deploy/k8s/grafana-configmap.yaml \ + -f deploy/k8s/grafana-deployment.yaml \ + -f deploy/k8s/grafana-service.yaml +``` + +#### 6.6 状态检查 + +```bash +# 查看所有 Pod 状态 +kubectl get pods + +# 查看所有 Service 及 NodePort +kubectl get svc +``` + +#### 6.7 端口汇总 + +| 服务 | NodePort | 访问地址 | 说明 | +| :--- | :--- | :--- | :--- | +| **Jaeger UI** | `31686` | `http://:31686` | 链路追踪查询界面 | +| **Loki** | `31100` | `http://:31100` | 日志 HTTP API | +| **Grafana** | `31000` | `http://:31000` | 可视化界面，账号 `admin / coslight` | +| **OTLP gRPC** | `31317` | `:31317` | ModelRT OTel 上报地址（gRPC） | +| **OTLP HTTP** | `31318` | `http://:31318` | ModelRT OTel 上报地址（HTTP） | + +#### 6.8 清理 + +```bash +kubectl delete -f deploy/k8s/ +``` + +### 7\. Mac 本地访问（SSH 隧道） + +`ModelRT / EventRT` 在 `Mac` 本地运行时，依赖的 `RabbitMQ`、`Redis`、`Jaeger`、`Loki`、`Grafana` 均部署在 `Ubuntu` 宿主机（`192.168.1.101`）上的 `Minikube`（`192.168.49.2`）中。由于 `Minikube` 网络不直接对外暴露，需通过 `SSH` 本地端口转发建立访问隧道。 + +#### 7.1 网络拓扑 + +``` text +Mac 本地端口 ──SSH隧道──▶ Ubuntu 宿主机 (192.168.1.101) ──▶ Minikube NodePort (192.168.49.2) +``` + +#### 7.2 建立隧道 + +```bash +ssh -L 5432:192.168.49.2:30432 \ + -L 27017:192.168.49.2:30017 \ + -L 5671:192.168.49.2:30671 \ + -L 15671:192.168.49.2:31671 \ + -L 6379:192.168.49.2:30001 \ + -L 4318:192.168.49.2:31318 \ + -L 16686:192.168.49.2:31686 \ + -L 3100:192.168.49.2:31100 \ + -L 3000:192.168.49.2:31000 \ + douxu@192.168.1.101 +``` + +如需后台静默运行（不占用终端）： + +```bash +ssh -fN \ + -L 5432:192.168.49.2:30432 \ + -L 27017:192.168.49.2:30017 \ + -L 5671:192.168.49.2:30671 \ + -L 15671:192.168.49.2:31671 \ + -L 6379:192.168.49.2:30001 \ + -L 4318:192.168.49.2:31318 \ + -L 16686:192.168.49.2:31686 \ + -L 3100:192.168.49.2:31100 \ + -L 3000:192.168.49.2:31000 \ + douxu@192.168.1.101 +``` + +#### 7.3 端口映射说明 + +| Mac 本地端口 | Minikube NodePort | 服务 | 说明 | +| :--- | :--- | :--- | :--- | +| `5432` | `30432` | PostgreSQL | 数据库连接 `localhost:5432` | +| `27017` | `30017` | MongoDB | 数据库连接 `localhost:27017` | +| `5671` | `30671` | RabbitMQ AMQP | ModelRT / EventRT 消息队列连接 | +| `15671` | `31671` | RabbitMQ Management | RabbitMQ 管理界面 `http://localhost:15671` | +| `6379` | `30001` | Redis | 分布式锁 / 数据存储 | +| `4318` | `31318` | OTLP HTTP | OTel Trace 上报（Jaeger Collector） | +| `16686` | `31686` | Jaeger UI | 链路追踪查询 `http://localhost:16686` | +| `3100` | `31100` | Loki | 日志查询 API | +| `3000` | `31000` | Grafana | 可视化界面 `http://localhost:3000` | + +> **注意：** 隧道建立后，本地配置文件中所有服务地址均填 `localhost:<本地端口>`，无需修改即可在 `Mac` 上直接运行服务。 + +#### 7.4 关闭隧道 + +前台运行时直接 `Ctrl+C`；后台运行时查找并终止进程： + +```bash +# 找到 ssh 隧道进程 +ps aux | grep "ssh -fN" +# 终止（替换为实际 PID） +kill +``` + +### 8\. 后续操作（停止与清理） + +#### 8.1 停止容器 ```bash docker stop postgres redis ``` -#### 4.2 删除容器（删除后数据将丢失） +#### 8.2 删除容器（删除后数据将丢失） ```bash docker rm postgres redis diff --git a/deploy/dockerfile/modelrt.Dockerfile b/deploy/dockerfile/modelrt.Dockerfile index e251642..4f83bb4 100644 --- a/deploy/dockerfile/modelrt.Dockerfile +++ b/deploy/dockerfile/modelrt.Dockerfile @@ -1,19 +1,35 @@ -FROM golang:1.24-alpine AS builder +FROM golang:1.25-alpine AS builder +RUN apk --no-cache upgrade WORKDIR /app -COPY go.mod . -COPY go.sum . +COPY go.mod go.sum ./ RUN GOPROXY="https://goproxy.cn,direct" go mod download COPY . . -RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o modelrt main.go +RUN CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w" \ + -trimpath \ + -mod=readonly \ + -o modelrt main.go -FROM alpine:latest -WORKDIR /app +# Prepare runtime dependencies in a pinned Alpine stage so they can be +# copied into scratch without pulling any vulnerable OS packages at run time. +FROM alpine:3.21 AS certs ARG USER_ID=1000 -RUN adduser -D -u ${USER_ID} modelrt +RUN apk --no-cache add ca-certificates tzdata && \ + adduser -D -u ${USER_ID} modelrt + +FROM scratch +# CA certificates required for TLS connections (RabbitMQ amqps://) +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +# Timezone data +COPY --from=certs /usr/share/zoneinfo /usr/share/zoneinfo +# Non-root user/group definitions +COPY --from=certs /etc/passwd /etc/passwd +COPY --from=certs /etc/group /etc/group + +WORKDIR /app COPY --from=builder /app/modelrt ./modelrt COPY configs/config.example.yaml ./configs/config.example.yaml -RUN chown -R modelrt:modelrt /app -RUN chmod +x /app/modelrt + USER modelrt -CMD ["/app/modelrt", "-modelRT_config_dir=/app/configs"] \ No newline at end of file +CMD ["/app/modelrt", "-modelRT_config_dir=/app/configs"] diff --git a/deploy/k8s/grafana-configmap.yaml b/deploy/k8s/grafana-configmap.yaml new file mode 100644 index 0000000..76b2cb0 --- /dev/null +++ b/deploy/k8s/grafana-configmap.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: default +data: + datasources.yaml: | + apiVersion: 1 + datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true + jsonData: + # derivedFields: 从日志的 traceID 字段生成跳转链接到 Jaeger + derivedFields: + - matcherRegex: '"traceID":\s*"([a-f0-9]+)"' + name: TraceID + url: http://127.0.0.1:16686/trace/$${__value.raw} + targetBlank: true + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://jaeger:16686 diff --git a/deploy/k8s/grafana-deployment.yaml b/deploy/k8s/grafana-deployment.yaml new file mode 100644 index 0000000..9f23045 --- /dev/null +++ b/deploy/k8s/grafana-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:10.4.2 + ports: + - containerPort: 3000 + env: + - name: GF_SECURITY_ADMIN_USER + value: "coslight" + - name: GF_SECURITY_ADMIN_PASSWORD + value: "coslight@tj" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: datasources + configMap: + name: grafana-datasources diff --git a/deploy/k8s/grafana-service.yaml b/deploy/k8s/grafana-service.yaml new file mode 100644 index 0000000..1cc3782 --- /dev/null +++ b/deploy/k8s/grafana-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: default +spec: + ports: + - name: http + port: 3000 + targetPort: 3000 + nodePort: 31000 # Grafana UI: http://:31000 + selector: + app: grafana + type: NodePort diff --git a/deploy/k8s/jaeger-deployment.yaml b/deploy/k8s/jaeger-deployment.yaml new file mode 100644 index 0000000..c0444b7 --- /dev/null +++ b/deploy/k8s/jaeger-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: jaeger + template: + metadata: + labels: + app: jaeger + spec: + containers: + - name: jaeger + image: jaegertracing/all-in-one:1.56 + env: + - name: COLLECTOR_OTLP_ENABLED + value: "true" + ports: + - containerPort: 16686 # UI + - containerPort: 14268 # Jaeger Collector + - containerPort: 4317 # OTLP gRPC + - containerPort: 4318 # OTLP HTTP + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi diff --git a/deploy/k8s/jaeger-service.yaml b/deploy/k8s/jaeger-service.yaml new file mode 100644 index 0000000..d1e4779 --- /dev/null +++ b/deploy/k8s/jaeger-service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: jaeger + labels: + app: jaeger +spec: + ports: + - name: ui + port: 16686 + targetPort: 16686 + nodePort: 31686 # Jaeger UI，浏览器访问 http://:31686 + - name: collector-http + port: 14268 + targetPort: 14268 + nodePort: 31268 # Jaeger 原生 HTTP collector（非 OTel） + - name: otlp-http + port: 4318 + targetPort: 4318 + nodePort: 31318 # OTLP HTTP，集群外使用 :31318 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + nodePort: 31317 # OTLP gRPC，集群外使用 :31317 + selector: + app: jaeger + type: NodePort diff --git a/deploy/k8s/loki-configmap.yaml b/deploy/k8s/loki-configmap.yaml new file mode 100644 index 0000000..b126a9f --- /dev/null +++ b/deploy/k8s/loki-configmap.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-config + namespace: default +data: + loki.yaml: | + auth_enabled: false + + server: + http_listen_port: 3100 + + ingester: + wal: + enabled: true + dir: /loki/wal # 指向 PVC 挂载路径，避免在容器根目录创建 /wal 时 permission denied + lifecycler: + ring: + kvstore: + store: inmemory + replication_factor: 1 + chunk_idle_period: 5m + chunk_retain_period: 30s + + schema_config: + configs: + - from: 2024-01-01 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + + storage_config: + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/cache + shared_store: filesystem + filesystem: + directory: /loki/chunks + + limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + + compactor: + working_directory: /loki/compactor + shared_store: filesystem diff --git a/deploy/k8s/loki-deployment.yaml b/deploy/k8s/loki-deployment.yaml new file mode 100644 index 0000000..63ff925 --- /dev/null +++ b/deploy/k8s/loki-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: loki + template: + metadata: + labels: + app: loki + spec: + securityContext: + fsGroup: 10001 # 使 PVC 挂载目录对 Loki 默认用户（UID 10001）可写 + runAsUser: 10001 + runAsGroup: 10001 + containers: + - name: loki + image: grafana/loki:2.9.4 + args: + - -config.file=/etc/loki/loki.yaml + ports: + - containerPort: 3100 + volumeMounts: + - name: config + mountPath: /etc/loki + - name: storage + mountPath: /loki + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: config + configMap: + name: loki-config + - name: storage + persistentVolumeClaim: + claimName: loki-pvc diff --git a/deploy/k8s/loki-pvc.yaml b/deploy/k8s/loki-pvc.yaml new file mode 100644 index 0000000..f329a08 --- /dev/null +++ b/deploy/k8s/loki-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: loki-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/deploy/k8s/loki-service.yaml b/deploy/k8s/loki-service.yaml new file mode 100644 index 0000000..e0df759 --- /dev/null +++ b/deploy/k8s/loki-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: loki + namespace: default +spec: + ports: + - name: http + port: 3100 + targetPort: 3100 + nodePort: 31100 # 集群外访问: http://:31100 + selector: + app: loki + type: NodePort diff --git a/deploy/k8s/modelrt-certs-secret.sh b/deploy/k8s/modelrt-certs-secret.sh new file mode 100644 index 0000000..33a1bbd --- /dev/null +++ b/deploy/k8s/modelrt-certs-secret.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Create the modelrt client certificate secret. +# Run this script from the directory that contains the three cert files, +# or adjust the paths below to point at the actual files. +# +# Expected files (generated during RabbitMQ TLS setup): +# ca_certificate.pem +# modelrt_client_cert.pem +# modelrt_client_key.pem + +kubectl create secret generic modelrt-certs \ + --from-file=ca_certificate.pem=./ca_certificate.pem \ + --from-file=modelrt_client_cert.pem=./modelrt_client_cert.pem \ + --from-file=modelrt_client_key.pem=./modelrt_client_key.pem diff --git a/deploy/k8s/modelrt-configmap.yaml b/deploy/k8s/modelrt-configmap.yaml new file mode 100644 index 0000000..b1a6afc --- /dev/null +++ b/deploy/k8s/modelrt-configmap.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: modelrt-config +data: + config.yaml: | + postgres: + host: "192.168.1.101" + port: 5432 + database: "demo" + user: "postgres" + password: "" # injected via env POSTGRES_PASSWORD + + rabbitmq: + ca_cert_path: "/app/configs/certs/ca_certificate.pem" + client_key_path: "/app/configs/certs/modelrt_client_key.pem" + client_key_password: "" + client_cert_path: "/app/configs/certs/modelrt_client_cert.pem" + insecure_skip_verify: false + server_name: "rabbitmq-server" + user: "" + password: "" + host: "rabbitmq-service" + port: 5671 + + logger: + mode: "production" + level: "info" + filepath: "" + maxsize: 100 + maxbackups: 5 + maxage: 30 + compress: false + loki: + endpoint: "" # Promtail handles log collection in K8s, direct push disabled + + otel: + endpoint: "jaeger:4318" + insecure: true + + ants: + parse_concurrent_quantity: 10 + rtd_receive_concurrent_quantity: 10 + + async_task: + worker_pool_size: 10 + queue_consumer_count: 2 + max_retry_count: 3 + retry_initial_delay: 1s + retry_max_delay: 5m + health_check_interval: 30s + + locker_redis: + addr: "redis-service:6379" + password: "" + db: 1 + poolsize: 50 + dial_timeout: 10 + read_timeout: 10 + write_timeout: 10 + + storage_redis: + addr: "redis-service:6379" + password: "" + db: 0 + poolsize: 50 + dial_timeout: 10 + read_timeout: 10 + write_timeout: 10 + + base: + grid_id: 1 + zone_id: 1 + station_id: 1 + + service: + service_addr: ":8080" + service_name: "modelRT" + secret_key: "" # injected via env SERVICE_SECRET_KEY + deploy_env: "production" + + dataRT: + host: "http://127.0.0.1" + port: 8888 + polling_api: "datart/getPointData" + polling_api_method: "GET" diff --git a/deploy/k8s/modelrt-deployment.yaml b/deploy/k8s/modelrt-deployment.yaml new file mode 100644 index 0000000..38a9a23 --- /dev/null +++ b/deploy/k8s/modelrt-deployment.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: modelrt + labels: + app: modelrt +spec: + replicas: 1 + selector: + matchLabels: + app: modelrt + template: + metadata: + labels: + app: modelrt + spec: + containers: + - name: modelrt + image: coslight/modelrt:latest + imagePullPolicy: IfNotPresent + args: + - "-modelRT_config_dir=/app/configs" + - "-modelRT_config_name=config" + - "-modelRT_config_type=yaml" + ports: + - containerPort: 8080 + env: + # Downward API — injected into every log line by logger/zap.go containerFields() + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # HOSTNAME is set automatically by K8s to the pod name + # Sensitive values injected from Secret so they stay out of ConfigMap + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: modelrt-secret + key: postgres-password + - name: SERVICE_SECRET_KEY + valueFrom: + secretKeyRef: + name: modelrt-secret + key: secret-key + volumeMounts: + - name: config + mountPath: /app/configs/config.yaml + subPath: config.yaml + readOnly: true + - name: certs + mountPath: /app/configs/certs + readOnly: true + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + runAsUser: 1000 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + livenessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + volumes: + - name: config + configMap: + name: modelrt-config + - name: certs + secret: + secretName: modelrt-certs diff --git a/deploy/k8s/modelrt-secret.yaml b/deploy/k8s/modelrt-secret.yaml new file mode 100644 index 0000000..e078e4d --- /dev/null +++ b/deploy/k8s/modelrt-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: modelrt-secret +type: Opaque +stringData: + postgres-password: "coslight" + secret-key: "modelrt_key" diff --git a/deploy/k8s/modelrt-service.yaml b/deploy/k8s/modelrt-service.yaml new file mode 100644 index 0000000..88b2cf7 --- /dev/null +++ b/deploy/k8s/modelrt-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: modelrt-service + labels: + app: modelrt +spec: + type: NodePort + selector: + app: modelrt + ports: + - name: http + port: 8080 + targetPort: 8080 + nodePort: 30080 diff --git a/deploy/k8s/mongodb-pvc.yaml b/deploy/k8s/mongodb-pvc.yaml new file mode 100644 index 0000000..d009b0a --- /dev/null +++ b/deploy/k8s/mongodb-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongodb-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/deploy/k8s/mongodb-secret.yaml b/deploy/k8s/mongodb-secret.yaml new file mode 100644 index 0000000..53363f2 --- /dev/null +++ b/deploy/k8s/mongodb-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mongodb-secret +type: Opaque +stringData: + MONGO_INITDB_ROOT_USERNAME: admin + MONGO_INITDB_ROOT_PASSWORD: coslight diff --git a/deploy/k8s/mongodb-service.yaml b/deploy/k8s/mongodb-service.yaml new file mode 100644 index 0000000..daf946a --- /dev/null +++ b/deploy/k8s/mongodb-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongodb + labels: + app: mongodb +spec: + type: NodePort + selector: + app: mongodb + ports: + - name: mongodb + port: 27017 + targetPort: 27017 + nodePort: 30017 diff --git a/deploy/k8s/mongodb-statefulset.yaml b/deploy/k8s/mongodb-statefulset.yaml new file mode 100644 index 0000000..8d1de21 --- /dev/null +++ b/deploy/k8s/mongodb-statefulset.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mongodb + labels: + app: mongodb +spec: + serviceName: mongodb + replicas: 1 + selector: + matchLabels: + app: mongodb + template: + metadata: + labels: + app: mongodb + spec: + containers: + - name: mongodb + image: mongo:7.0 + imagePullPolicy: IfNotPresent + ports: + - name: mongodb + containerPort: 27017 + envFrom: + - secretRef: + name: mongodb-secret + volumeMounts: + - name: mongodb-data + mountPath: /data/db + readinessProbe: + exec: + command: + - mongosh + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 12 + livenessProbe: + exec: + command: + - mongosh + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumes: + - name: mongodb-data + persistentVolumeClaim: + claimName: mongodb-data diff --git a/deploy/k8s/pg-configmap.yaml b/deploy/k8s/pg-configmap.yaml new file mode 100644 index 0000000..1daf181 --- /dev/null +++ b/deploy/k8s/pg-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config +data: + POSTGRES_DB: demo + POSTGRES_USER: postgres + POSTGRES_PASSWORD: coslight diff --git a/deploy/k8s/pg-pvc.yaml b/deploy/k8s/pg-pvc.yaml new file mode 100644 index 0000000..f172ce7 --- /dev/null +++ b/deploy/k8s/pg-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/deploy/k8s/pg-service.yaml b/deploy/k8s/pg-service.yaml new file mode 100644 index 0000000..5e11fe0 --- /dev/null +++ b/deploy/k8s/pg-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + type: NodePort + selector: + app: postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 + nodePort: 30432 diff --git a/deploy/k8s/pg-statefulset.yaml b/deploy/k8s/pg-statefulset.yaml new file mode 100644 index 0000000..bdf0ec8 --- /dev/null +++ b/deploy/k8s/pg-statefulset.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + labels: + app: postgres +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13.16 + imagePullPolicy: IfNotPresent + ports: + - name: postgres + containerPort: 5432 + envFrom: + - configMapRef: + name: postgres-config + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + readinessProbe: + exec: + command: + - sh + - -c + - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" + initialDelaySeconds: 8 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 12 + livenessProbe: + exec: + command: + - sh + - -c + - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-data diff --git a/deploy/k8s/promtail-configmap.yaml b/deploy/k8s/promtail-configmap.yaml new file mode 100644 index 0000000..0ccf089 --- /dev/null +++ b/deploy/k8s/promtail-configmap.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: promtail-config + namespace: default +data: + promtail.yaml: | + server: + http_listen_port: 9080 + grpc_listen_port: 0 + + positions: + filename: /tmp/positions.yaml + + clients: + - url: http://loki:3100/loki/api/v1/push + + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + pipeline_stages: + # 解析 zap 输出的 JSON 日志，提取结构化字段 + - json: + expressions: + level: level + traceID: traceID + spanID: spanID + caller: caller + pod: pod + namespace: namespace + node: node + # 将关键字段提升为 Loki Label,支持在 Grafana 中按实例/Trace 过滤 + - labels: + level: + traceID: + pod: + namespace: + node: + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - source_labels: [__meta_kubernetes_pod_label_app] + target_label: app + # 只采集有 app label 的 Pod + - source_labels: [__meta_kubernetes_pod_label_app] + action: keep + regex: .+ diff --git a/deploy/k8s/promtail-daemonset.yaml b/deploy/k8s/promtail-daemonset.yaml new file mode 100644 index 0000000..eedf72d --- /dev/null +++ b/deploy/k8s/promtail-daemonset.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: promtail + namespace: default +spec: + selector: + matchLabels: + app: promtail + template: + metadata: + labels: + app: promtail + spec: + serviceAccountName: promtail + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: promtail + image: grafana/promtail:2.9.4 + args: + - -config.file=/etc/promtail/promtail.yaml + ports: + - containerPort: 9080 + volumeMounts: + - name: config + mountPath: /etc/promtail + - name: varlog + mountPath: /var/log + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi + volumes: + - name: config + configMap: + name: promtail-config + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/deploy/k8s/promtail-rbac.yaml b/deploy/k8s/promtail-rbac.yaml new file mode 100644 index 0000000..2433f23 --- /dev/null +++ b/deploy/k8s/promtail-rbac.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: promtail + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: promtail +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/proxy", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: promtail +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: promtail +subjects: + - kind: ServiceAccount + name: promtail + namespace: default diff --git a/deploy/k8s/rabbitmq-config.yaml b/deploy/k8s/rabbitmq-config.yaml new file mode 100644 index 0000000..a5cbad7 --- /dev/null +++ b/deploy/k8s/rabbitmq-config.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-config +data: + rabbitmq.conf: | + # 确保允许PLAIN认证 + auth_mechanisms.1 = PLAIN + auth_mechanisms.2 = AMQPLAIN + auth_mechanisms.3 = EXTERNAL + # 允许admin用户通过远程方式连接 + loopback_users.admin = false + # 默认心跳和监听配置可在此扩展 + # 确定 ssl 连接时验证使用的用户名 + ssl_cert_login_from = common_name + # 开启此项配置会导致只能通过TLS端口访问 + listeners.tcp = none + listeners.ssl.default = 5671 + # default user config + load_definitions = /etc/rabbitmq/definitions.json + # ssl config + ssl_options.cacertfile = /etc/rabbitmq/certs/ca_certificate.pem + ssl_options.certfile = /etc/rabbitmq/certs/server_certificate.pem + ssl_options.keyfile = /etc/rabbitmq/certs/server_key.pem + ssl_options.verify = verify_peer + ssl_options.fail_if_no_peer_cert = true + # management config + management.ssl.port = 15671 + management.ssl.cacertfile = /etc/rabbitmq/certs/ca_certificate.pem + management.ssl.certfile = /etc/rabbitmq/certs/server_certificate.pem + management.ssl.keyfile = /etc/rabbitmq/certs/server_key.pem + management.ssl.verify = verify_peer + management.ssl.fail_if_no_peer_cert = true diff --git a/deploy/k8s/rabbitmq-deployment.yaml b/deploy/k8s/rabbitmq-deployment.yaml new file mode 100644 index 0000000..758daca --- /dev/null +++ b/deploy/k8s/rabbitmq-deployment.yaml @@ -0,0 +1,81 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: eventrt-rabbitmq +spec: + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:4.1.1-management-alpine + ports: + - containerPort: 4369 + - containerPort: 5671 + - containerPort: 5672 # AMQP + - containerPort: 15671 + - containerPort: 15672 # Management UI + - containerPort: 15691 + - containerPort: 15692 + - containerPort: 25672 + env: + - name: RABBITMQ_DEFAULT_USER + valueFrom: + secretKeyRef: + name: rabbitmq-secret + key: rabbitmq-user + - name: RABBITMQ_DEFAULT_PASS + valueFrom: + secretKeyRef: + name: rabbitmq-secret + key: rabbitmq-pass + - name: RABBITMQ_ERLANG_COOKIE + valueFrom: + secretKeyRef: + name: rabbitmq-secret + key: erlang-cookie + - name: RABBITMQ_DEFAULT_VHOST + value: "/" + volumeMounts: + - name: rabbitmq-certs-volume + mountPath: /etc/rabbitmq/certs + readOnly: true + - name: rabbitmq-config-volume + mountPath: /etc/rabbitmq/rabbitmq.conf + subPath: rabbitmq.conf + - name: rabbitmq-config-volume + mountPath: /etc/rabbitmq/advanced.config + subPath: advanced.config + readOnly: true + - name: plugins-config-volume + mountPath: /etc/rabbitmq/enabled_plugins + subPath: enabled_plugins + - name: users-config-volume + mountPath: /etc/rabbitmq/definitions.json + subPath: definitions.json + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + volumes: + - name: rabbitmq-certs-volume + secret: + secretName: rabbitmq-certs + - name: rabbitmq-config-volume + configMap: + name: rabbitmq-config + - name: rabbitmq-advanced-config-volume + configMap: + name: rabbitmq-config + - name: plugins-config-volume + configMap: + name: rabbit-plugins-conf + - name: users-config-volume + configMap: + name: rabbitmq-users-definitions + - name: rabbitmq-data + emptyDir: {} diff --git a/deploy/k8s/rabbitmq-secret.yaml b/deploy/k8s/rabbitmq-secret.yaml new file mode 100644 index 0000000..eae46a1 --- /dev/null +++ b/deploy/k8s/rabbitmq-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: rabbitmq-secret +type: Opaque +stringData: + rabbitmq-user: "coslight" + rabbitmq-pass: "coslight@tj" + erlang-cookie: "secret-erlang-cookie" diff --git a/deploy/k8s/rabbitmq-service.yaml b/deploy/k8s/rabbitmq-service.yaml new file mode 100644 index 0000000..6cdb259 --- /dev/null +++ b/deploy/k8s/rabbitmq-service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq-service +spec: + type: NodePort # 在 Minikube 中使用 NodePort 方便外部访问 + selector: + app: rabbitmq + ports: + - name: amqp-ssl + protocol: TCP + port: 5671 + targetPort: 5671 + nodePort: 30671 + - name: amqp + protocol: TCP + port: 5672 + targetPort: 5672 + nodePort: 30672 + - name: management-ssl + protocol: TCP + port: 15671 + targetPort: 15671 + nodePort: 31671 + - name: management + protocol: TCP + port: 15672 + targetPort: 15672 + nodePort: 31672 diff --git a/deploy/k8s/rabbitmq-users-config.yaml b/deploy/k8s/rabbitmq-users-config.yaml new file mode 100644 index 0000000..8de5f30 --- /dev/null +++ b/deploy/k8s/rabbitmq-users-config.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-users-definitions +data: + definitions.json: | + { + "users": [ + { + "name": "coslight", + "password_hash": "Gl2XVEJwPwDZQF8ZhsYnvm83wMkdftY3/raxyntdZueyx/Uv", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["administrator"] + }, + { + "name": "web-client", + "password_hash": "", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["management"] + }, + { + "name": "modelrt-client", + "password_hash": "", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["management"] + }, + { + "name": "eventrt-client", + "password_hash": "", + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["management"] + } + ], + "vhosts": [ { "name": "/" } ], + "permissions": [ + { + "user": "coslight", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "web-client", + "vhost": "/", + "configure": "^$", + "write": ".*", + "read": ".*" + }, + { + "user": "modelrt-client", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + }, + { + "user": "eventrt-client", + "vhost": "/", + "configure": ".*", + "write": ".*", + "read": ".*" + } + ], + "topic_permissions": [], + "parameters": [], + "global_parameters": [ + { + "name": "cluster_name", + "value": "evnetrt-rabbitmq-cluster" + } + ], + "policies": [], + "queues": [], + "exchanges": [], + "bindings": [] + } diff --git a/deploy/k8s/redis-deployment.yaml b/deploy/k8s/redis-deployment.yaml new file mode 100644 index 0000000..b2f08fc --- /dev/null +++ b/deploy/k8s/redis-deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis/redis-stack-server:latest + resources: + limits: + memory: "128Mi" + cpu: "500m" + ports: + - containerPort: 6379 diff --git a/deploy/k8s/redis-service.yaml b/deploy/k8s/redis-service.yaml new file mode 100644 index 0000000..ba82d15 --- /dev/null +++ b/deploy/k8s/redis-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-service +spec: + type: NodePort + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 + nodePort: 30001 + diff --git a/deploy/redis-test-data/real-time-subpull/sub_data_injection.go b/deploy/redis-test-data/real-time-subpull/sub_data_injection.go index a15cef6..4c0728b 100644 --- a/deploy/redis-test-data/real-time-subpull/sub_data_injection.go +++ b/deploy/redis-test-data/real-time-subpull/sub_data_injection.go @@ -129,9 +129,9 @@ func generateOutlierSegments(totalSize, minLength, maxLength, count int, distrib segments := make([]OutlierSegment, 0, count) usedPositions := make(map[int]bool) - for i := 0; i < count; i++ { + for range count { // 尝试多次寻找合适的位置 - for attempt := 0; attempt < 10; attempt++ { + for range 10 { length := rand.Intn(maxLength-minLength+1) + minLength start := rand.Intn(totalSize - length) diff --git a/diagram/anchor_set.go b/diagram/anchor_set.go index 94a83f4..a6447ca 100644 --- a/diagram/anchor_set.go +++ b/diagram/anchor_set.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -31,11 +32,9 @@ func UpdateAnchorValue(componentUUID string, anchorValue string) bool { // StoreAnchorValue define func of store anchor value with componentUUID and anchor name func StoreAnchorValue(componentUUID string, anchorValue string) { anchorValueOverview.Store(componentUUID, anchorValue) - return } // DeleteAnchorValue define func of delete anchor value with componentUUID func DeleteAnchorValue(componentUUID string) { anchorValueOverview.Delete(componentUUID) - return } diff --git a/diagram/component_set.go b/diagram/component_set.go index 7a7b6c5..da9bddf 100644 --- a/diagram/component_set.go +++ b/diagram/component_set.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -33,11 +34,9 @@ func UpdateComponentMap(componentID int64, componentInfo *orm.Component) bool { // StoreComponentMap define func of store circuit diagram data with component uuid and component info func StoreComponentMap(componentUUID string, componentInfo *orm.Component) { diagramsOverview.Store(componentUUID, componentInfo) - return } // DeleteComponentMap define func of delete circuit diagram data with component uuid func DeleteComponentMap(componentUUID string) { diagramsOverview.Delete(componentUUID) - return } diff --git a/diagram/graph.go b/diagram/graph.go index 1d9ee6e..6c4970d 100644 --- a/diagram/graph.go +++ b/diagram/graph.go @@ -112,7 +112,6 @@ func (g *Graph) DelEdge(from, to uuid.UUID) error { return fmt.Errorf("delete edge failed: %w", err) } - fmt.Println("fromKeys:", fromKeys) for _, fromUUID := range fromKeys { fromKey := fromUUID.String() var delIndex int diff --git a/diagram/hash_test.go b/diagram/hash_test.go index ed320f3..5b2053f 100644 --- a/diagram/hash_test.go +++ b/diagram/hash_test.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -17,7 +18,7 @@ func TestHMSet(t *testing.T) { PoolSize: 50, DialTimeout: 10 * time.Second, }) - params := map[string]interface{}{ + params := map[string]any{ "field1": "Hello1", "field2": "World1", "field3": 11, @@ -29,5 +30,4 @@ func TestHMSet(t *testing.T) { fmt.Printf("err:%v\n", err) } fmt.Printf("res:%v\n", res) - return } diff --git a/diagram/multi_branch_tree.go b/diagram/multi_branch_tree.go index ceb6cfa..d61e385 100644 --- a/diagram/multi_branch_tree.go +++ b/diagram/multi_branch_tree.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -52,7 +53,7 @@ func (n *MultiBranchTreeNode) FindNodeByID(id uuid.UUID) *MultiBranchTreeNode { } func (n *MultiBranchTreeNode) PrintTree(level int) { - for i := 0; i < level; i++ { + for range level { fmt.Print(" ") } @@ -62,3 +63,63 @@ func (n *MultiBranchTreeNode) PrintTree(level int) { child.PrintTree(level + 1) } } + +// FindPath returns the ordered node sequence from startID to endID using the +// supplied nodeMap for O(1) lookup. It walks each node up to the root to find +// the LCA, then stitches the two half-paths together. +// Returns nil when either node is absent from nodeMap or no path exists. +func FindPath(startID, endID uuid.UUID, nodeMap map[uuid.UUID]*MultiBranchTreeNode) []*MultiBranchTreeNode { + startNode, ok := nodeMap[startID] + if !ok { + return nil + } + endNode, ok := nodeMap[endID] + if !ok { + return nil + } + + // collect ancestors (inclusive) from a node up to the root sentinel + ancestors := func(n *MultiBranchTreeNode) []*MultiBranchTreeNode { + var chain []*MultiBranchTreeNode + for n != nil { + chain = append(chain, n) + n = n.Parent + } + return chain + } + + startChain := ancestors(startNode) // [start, ..., root] + endChain := ancestors(endNode) // [end, ..., root] + + // index startChain by ID for fast LCA detection + startIdx := make(map[uuid.UUID]int, len(startChain)) + for i, node := range startChain { + startIdx[node.ID] = i + } + + // find LCA: first node in endChain that also appears in startChain + lcaEndPos := -1 + lcaStartPos := -1 + for i, node := range endChain { + if j, found := startIdx[node.ID]; found { + lcaEndPos = i + lcaStartPos = j + break + } + } + + if lcaEndPos < 0 { + return nil // disconnected + } + + // path = startChain[0..lcaStartPos] reversed + endChain[lcaEndPos..0] reversed + path := make([]*MultiBranchTreeNode, 0, lcaStartPos+lcaEndPos+1) + for i := 0; i <= lcaStartPos; i++ { + path = append(path, startChain[i]) + } + // append end-side (skip LCA to avoid duplication), reversed + for i := lcaEndPos - 1; i >= 0; i-- { + path = append(path, endChain[i]) + } + return path +} diff --git a/diagram/redis_client.go b/diagram/redis_client.go index 283669d..4b673a8 100644 --- a/diagram/redis_client.go +++ b/diagram/redis_client.go @@ -19,8 +19,8 @@ func NewRedisClient() *RedisClient { } } -// QueryByZRangeByLex define func to query real time data from redis zset -func (rc *RedisClient) QueryByZRangeByLex(ctx context.Context, key string, size int64) ([]redis.Z, error) { +// QueryByZRange define func to query real time data from redis zset +func (rc *RedisClient) QueryByZRange(ctx context.Context, key string, size int64) ([]redis.Z, error) { client := rc.Client args := redis.ZRangeArgs{ Key: key, diff --git a/diagram/redis_hash.go b/diagram/redis_hash.go index 9ef0f22..2382828 100644 --- a/diagram/redis_hash.go +++ b/diagram/redis_hash.go @@ -29,7 +29,7 @@ func NewRedisHash(ctx context.Context, hashKey string, lockLeaseTime uint64, nee } // SetRedisHashByMap define func of set redis hash by map struct -func (rh *RedisHash) SetRedisHashByMap(fields map[string]interface{}) error { +func (rh *RedisHash) SetRedisHashByMap(fields map[string]any) error { err := rh.rwLocker.WLock(rh.ctx) if err != nil { logger.Error(rh.ctx, "lock wLock by hash_key failed", "hash_key", rh.hashKey, "error", err) @@ -46,7 +46,7 @@ func (rh *RedisHash) SetRedisHashByMap(fields map[string]interface{}) error { } // SetRedisHashByKV define func of set redis hash by kv struct -func (rh *RedisHash) SetRedisHashByKV(field string, value interface{}) error { +func (rh *RedisHash) SetRedisHashByKV(field string, value any) error { err := rh.rwLocker.WLock(rh.ctx) if err != nil { logger.Error(rh.ctx, "lock wLock by hash_key failed", "hash_key", rh.hashKey, "error", err) diff --git a/diagram/redis_init.go b/diagram/redis_init.go index d1d7a22..639f9f0 100644 --- a/diagram/redis_init.go +++ b/diagram/redis_init.go @@ -16,13 +16,15 @@ var ( ) // initClient define func of return successfully initialized redis client -func initClient(rCfg config.RedisConfig) *redis.Client { +func initClient(rCfg config.RedisConfig, deployEnv string) *redis.Client { client, err := util.NewRedisClient( rCfg.Addr, - util.WithPassword(rCfg.Password), + util.WithPassword(rCfg.Password, deployEnv), util.WithDB(rCfg.DB), util.WithPoolSize(rCfg.PoolSize), - util.WithTimeout(time.Duration(rCfg.Timeout)*time.Second), + util.WithConnectTimeout(time.Duration(rCfg.DialTimeout)*time.Second), + util.WithReadTimeout(time.Duration(rCfg.ReadTimeout)*time.Second), + util.WithWriteTimeout(time.Duration(rCfg.WriteTimeout)*time.Second), ) if err != nil { panic(err) @@ -31,9 +33,9 @@ func initClient(rCfg config.RedisConfig) *redis.Client { } // InitRedisClientInstance define func of return instance of redis client -func InitRedisClientInstance(rCfg config.RedisConfig) *redis.Client { +func InitRedisClientInstance(rCfg config.RedisConfig, deployEnv string) *redis.Client { once.Do(func() { - _globalStorageClient = initClient(rCfg) + _globalStorageClient = initClient(rCfg, deployEnv) }) return _globalStorageClient } diff --git a/diagram/redis_string.go b/diagram/redis_string.go index 94b7bd9..130a89a 100644 --- a/diagram/redis_string.go +++ b/diagram/redis_string.go @@ -46,7 +46,7 @@ func (rs *RedisString) Get(stringKey string) (string, error) { } // Set define func of set the value of key -func (rs *RedisString) Set(stringKey string, value interface{}) error { +func (rs *RedisString) Set(stringKey string, value any) error { err := rs.rwLocker.WLock(rs.ctx) if err != nil { logger.Error(rs.ctx, "lock wLock by stringKey failed", "string_key", stringKey, "error", err) diff --git a/diagram/redis_zset.go b/diagram/redis_zset.go index 549d28b..d350b4f 100644 --- a/diagram/redis_zset.go +++ b/diagram/redis_zset.go @@ -30,7 +30,7 @@ func NewRedisZSet(ctx context.Context, key string, lockLeaseTime uint64, needRef } // ZADD define func of add redis zset by members -func (rs *RedisZSet) ZADD(setKey string, score float64, member interface{}) error { +func (rs *RedisZSet) ZADD(setKey string, score float64, member any) error { err := rs.rwLocker.WLock(rs.ctx) if err != nil { logger.Error(rs.ctx, "lock wLock by setKey failed", "set_key", setKey, "error", err) diff --git a/diagram/topologic_set.go b/diagram/topologic_set.go index 9dbbec3..ccc17a9 100644 --- a/diagram/topologic_set.go +++ b/diagram/topologic_set.go @@ -1,3 +1,4 @@ +// Package diagram provide diagram data structure and operation package diagram import ( @@ -11,7 +12,7 @@ var graphOverview sync.Map // PrintGrapMap define func of print circuit diagram topologic info data func PrintGrapMap() { - graphOverview.Range(func(key, value interface{}) bool { + graphOverview.Range(func(key, value any) bool { fmt.Println(key, value) return true }) @@ -39,11 +40,9 @@ func UpdateGrapMap(pageID int64, graphInfo *Graph) bool { // StoreGraphMap define func of store circuit diagram topologic data with pageID and topologic info func StoreGraphMap(pageID int64, graphInfo *Graph) { graphOverview.Store(pageID, graphInfo) - return } // DeleteGraphMap define func of delete circuit diagram topologic data with pageID func DeleteGraphMap(pageID int64) { graphOverview.Delete(pageID) - return } diff --git a/distributedlock/locker_init.go b/distributedlock/locker_init.go index 35ecc78..507b00f 100644 --- a/distributedlock/locker_init.go +++ b/distributedlock/locker_init.go @@ -16,13 +16,15 @@ var ( ) // initClient define func of return successfully initialized redis client -func initClient(rCfg config.RedisConfig) *redis.Client { +func initClient(rCfg config.RedisConfig, deployEnv string) *redis.Client { client, err := util.NewRedisClient( rCfg.Addr, - util.WithPassword(rCfg.Password), + util.WithPassword(rCfg.Password, deployEnv), util.WithDB(rCfg.DB), util.WithPoolSize(rCfg.PoolSize), - util.WithTimeout(time.Duration(rCfg.Timeout)*time.Second), + util.WithConnectTimeout(time.Duration(rCfg.DialTimeout)*time.Second), + util.WithReadTimeout(time.Duration(rCfg.ReadTimeout)*time.Second), + util.WithWriteTimeout(time.Duration(rCfg.WriteTimeout)*time.Second), ) if err != nil { panic(err) @@ -31,9 +33,9 @@ func initClient(rCfg config.RedisConfig) *redis.Client { } // InitClientInstance define func of return instance of redis client -func InitClientInstance(rCfg config.RedisConfig) *redis.Client { +func InitClientInstance(rCfg config.RedisConfig, deployEnv string) *redis.Client { once.Do(func() { - _globalLockerClient = initClient(rCfg) + _globalLockerClient = initClient(rCfg, deployEnv) }) return _globalLockerClient } diff --git a/doc/async_task_api.md b/doc/async_task_api.md new file mode 100644 index 0000000..6a474ed --- /dev/null +++ b/doc/async_task_api.md @@ -0,0 +1,539 @@ +# ModelRT 异步任务 API 文档 + +## 1. 概述 + +ModelRT 异步任务系统基于 RabbitMQ 消息驱动，提供完整的任务生命周期管理。任务创建后进入队列，由后台 Worker 消费并执行，调用方可通过轮询接口获取进度和结果。 + +**Base URL**: `http://{host}:{port}`（默认 `localhost:8080`） + +**鉴权**: 公开接口需在 Header 中携带 `X-Service-Token`（由服务端启动时生成）。 + +--- + +## 2. API 端点总览 + +| 方法 | 路径 | 描述 | 权限 | +| :----- | :--------------------------------- | :--------------- | :----- | +| POST | `/task/async` | 创建异步任务 | 公开 | +| GET | `/task/async/results` | 批量查询任务结果 | 公开 | +| GET | `/task/async/{task_id}` | 查询单个任务详情 | 公开 | +| POST | `/task/async/{task_id}/cancel` | 取消异步任务 | 公开 | +| POST | `/task/internal/async/progress` | 更新任务进度 | 内部 | +| POST | `/task/internal/async/status` | 更新任务状态 | 内部 | + +--- + +## 3. 通用响应结构 + +### 成功响应 + +```json +{ + "code": 2000, + "msg": "success message", + "payload": {} +} +``` + +### 失败响应 + +```json +{ + "code": 4001, + "msg": "error description", + "payload": null +} +``` + +### 响应码说明 + +| code | 含义 | +| :--- | :--------------------- | +| 2000 | 成功 | +| 3000 | 处理失败 | +| 4001 | 请求参数无效 | +| 4002 | 未授权（Token 无效） | +| 5000 | 服务器内部错误 | + +--- + +## 4. 详细接口说明 + +### 4.1 创建异步任务 + +**POST** `/task/async` + +创建新的异步任务，任务进入队列等待 Worker 消费。返回 `task_id` 用于后续查询。 + +**请求头** + +``` +Content-Type: application/json +X-Service-Token: +``` + +**请求体** + +```json +{ + "task_type": "TOPOLOGY_ANALYSIS", + "params": { } +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :---------- | :----- | :--- | :-------------------------------------------------------------------------------- | +| `task_type` | string | 是 | 任务类型，枚举值见 §5.1 | +| `params` | object | 是 | 任务参数，不同任务类型的参数结构见 §5.3 | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task created successfully", + "payload": { + "task_id": "123e4567-e89b-12d3-a456-426614174000" + } +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------- | :--- | :-------------------------- | +| 参数格式错误 | 4001 | invalid request parameters | +| task_type 不合法 | 4001 | invalid task type | +| params 内容不合法 | 4001 | invalid task parameters | +| 数据库连接失败 | 5000 | database connection error | +| 任务写库失败 | 5000 | failed to create task | + +**curl 示例** + +```bash +curl -X POST "http://localhost:8080/task/async" \ + -H "Content-Type: application/json" \ + -H "X-Service-Token: " \ + -d '{ + "task_type": "TOPOLOGY_ANALYSIS", + "params": { + "start_component_uuid": "550e8400-e29b-41d4-a716-446655440000", + "end_component_uuid": "550e8400-e29b-41d4-a716-446655440001", + "check_in_service": true + } + }' +``` + +--- + +### 4.2 批量查询任务结果 + +**GET** `/task/async/results` + +根据一组任务 ID 批量查询状态和结果，适用于轮询多个任务。 + +**Query 参数** + +| 参数 | 类型 | 必填 | 说明 | +| :--------- | :----- | :--- | :----------------------------------- | +| `task_ids` | string | 是 | 逗号分隔的 UUID 列表，最少 1 个 | + +**请求示例** + +``` +GET /task/async/results?task_ids=123e4567-e89b-12d3-a456-426614174000,223e4567-e89b-12d3-a456-426614174001 +``` + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "query completed", + "payload": { + "total": 2, + "tasks": [ + { + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "task_type": "TOPOLOGY_ANALYSIS", + "status": "RUNNING", + "progress": 50, + "created_at": 1741846200 + }, + { + "task_id": "223e4567-e89b-12d3-a456-426614174001", + "task_type": "PERFORMANCE_ANALYSIS", + "status": "COMPLETED", + "progress": 100, + "created_at": 1741846200, + "finished_at": 1741846260, + "result": { + "components_analyzed": 5 + } + } + ] + } +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------- | :--- | :-------------------------- | +| 缺少 task_ids | 4001 | task_ids parameter is required | +| UUID 格式不合法 | 4001 | invalid task ID format | +| 数据库连接失败 | 5000 | database connection error | +| 查询失败 | 5000 | failed to query tasks | + +**curl 示例** + +```bash +curl "http://localhost:8080/task/async/results?task_ids=123e4567-e89b-12d3-a456-426614174000" +``` + +--- + +### 4.3 查询单个任务详情 + +**GET** `/task/async/{task_id}` + +查询单个任务的完整信息，包含结果或错误详情。 + +**路径参数** + +| 参数 | 类型 | 必填 | 说明 | +| :-------- | :----- | :--- | :-------------- | +| `task_id` | string | 是 | 任务 UUID | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "query completed", + "payload": { + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "task_type": "TOPOLOGY_ANALYSIS", + "status": "COMPLETED", + "progress": 100, + "created_at": 1741846200, + "finished_at": 1741846260, + "result": { + "path_exists": true, + "path_length": 3, + "path_nodes": ["comp-001", "comp-005", "comp-999"] + } + } +} +``` + +任务失败时 payload 附带错误信息： + +```json +{ + "code": 2000, + "msg": "query completed", + "payload": { + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "task_type": "TOPOLOGY_ANALYSIS", + "status": "FAILED", + "created_at": 1741846200, + "finished_at": 1741846210, + "error_code": 400102, + "error_message": "Component UUID not found", + "error_detail": { + "component_uuid": "550e8400-0000-0000-0000-000000000000" + } + } +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------- | :--- | :-------------------------- | +| 缺少 task_id | 4001 | task_id parameter is required | +| UUID 格式不合法 | 4001 | invalid task ID format | +| 任务不存在 | 404 | task not found | +| 数据库连接失败 | 5000 | database connection error | + +**curl 示例** + +```bash +curl "http://localhost:8080/task/async/123e4567-e89b-12d3-a456-426614174000" +``` + +--- + +### 4.4 取消异步任务 + +**POST** `/task/async/{task_id}/cancel` + +取消指定任务。**仅 `SUBMITTED`（排队中）状态的任务可以被取消**，已开始执行（`RUNNING`）或已结束的任务无法取消。 + +取消后任务状态变为 `FAILED`，错误码 `40003`，错误信息 `task cancelled by user`。 + +**路径参数** + +| 参数 | 类型 | 必填 | 说明 | +| :-------- | :----- | :--- | :-------- | +| `task_id` | string | 是 | 任务 UUID | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task cancelled successfully" +} +``` + +**失败响应** + +| 场景 | code | msg | +| :----------------------- | :--- | :--------------------------------------------------------- | +| 缺少 task_id | 400 | task_id parameter is required | +| UUID 格式不合法 | 400 | invalid task ID format | +| 任务不存在 | 404 | task not found | +| 任务已执行或已完成 | 400 | task cannot be cancelled (already running or completed) | +| 数据库连接失败 | 500 | database connection error | +| 取消操作失败 | 500 | failed to cancel task | + +**curl 示例** + +```bash +curl -X POST "http://localhost:8080/task/async/123e4567-e89b-12d3-a456-426614174000/cancel" +``` + +--- + +### 4.5 内部接口：更新任务进度 + +**POST** `/task/internal/async/progress` + +由 Worker 内部调用，更新任务执行进度（0-100）。 + +**请求体** + +```json +{ + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "progress": 75 +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------- | :----- | :--- | :---------------- | +| `task_id` | string | 是 | 任务 UUID | +| `progress` | int | 是 | 进度值，范围 0-100 | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task progress updated successfully", + "payload": null +} +``` + +--- + +### 4.6 内部接口：更新任务状态 + +**POST** `/task/internal/async/status` + +由 Worker 内部调用，更新任务状态。当状态更新为 `COMPLETED` 或 `FAILED` 时，同步写入 `finished_at` 时间戳。 + +**请求体** + +```json +{ + "task_id": "123e4567-e89b-12d3-a456-426614174000", + "status": "RUNNING", + "timestamp": 1741846205 +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :---------- | :----- | :--- | :--------------------------------------- | +| `task_id` | string | 是 | 任务 UUID | +| `status` | string | 是 | 目标状态，枚举值见 §5.2 | +| `timestamp` | int64 | 是 | 状态变更时间戳（Unix 秒） | + +**成功响应** `200` + +```json +{ + "code": 2000, + "msg": "task status updated successfully", + "payload": null +} +``` + +--- + +## 5. 数据结构参考 + +### 5.1 任务类型（task_type） + +| 枚举值 | 描述 | +| :--------------------- | :----------- | +| `TOPOLOGY_ANALYSIS` | 拓扑连通性分析 | +| `PERFORMANCE_ANALYSIS` | 性能分析 | +| `EVENT_ANALYSIS` | 事件分析 | +| `BATCH_IMPORT` | 批量数据导入 | +| `TEST` | 测试任务（系统验证用） | + +### 5.2 任务状态（status） + +| 枚举值 | 描述 | 可转换至 | +| :---------- | :----------------- | :------------------------------ | +| `SUBMITTED` | 已提交至队列 | `RUNNING`, `FAILED`（取消） | +| `RUNNING` | 正在执行 | `COMPLETED`, `FAILED` | +| `COMPLETED` | 执行成功 | — | +| `FAILED` | 执行失败或被取消 | — | + +### 5.3 各任务类型的 params 结构 + +#### TOPOLOGY_ANALYSIS — 拓扑连通性分析 + +分析两个元件之间是否存在连通路径。 + +```json +{ + "start_component_uuid": "550e8400-e29b-41d4-a716-446655440000", + "end_component_uuid": "550e8400-e29b-41d4-a716-446655440001", + "check_in_service": true +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------------- | :------ | :--- | :------------------------------------- | +| `start_component_uuid` | string | 是 | 起始元件 UUID | +| `end_component_uuid` | string | 是 | 目标元件 UUID | +| `check_in_service` | boolean | 否 | 是否只检查投运状态元件，默认 `true` | + +#### PERFORMANCE_ANALYSIS — 性能分析 + +```json +{ + "component_ids": ["comp-001", "comp-002"], + "time_range": { + "start": "2026-03-01T00:00:00Z", + "end": "2026-03-02T00:00:00Z" + } +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------- | :------- | :--- | :------------------ | +| `component_ids` | []string | 是 | 待分析的元件 ID 列表（至少 1 个） | +| `time_range` | object | 否 | 分析时间范围 | + +#### EVENT_ANALYSIS — 事件分析 + +```json +{ + "event_type": "MOTOR_START", + "start_time": "2026-03-01T00:00:00Z", + "end_time": "2026-03-02T00:00:00Z", + "components": ["comp-001", "comp-002"] +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :------------ | :------- | :--- | :------------------ | +| `event_type` | string | 是 | 事件类型 | +| `start_time` | string | 否 | 事件起始时间（RFC3339） | +| `end_time` | string | 否 | 事件截止时间（RFC3339） | +| `components` | []string | 否 | 关联的元件列表 | + +#### BATCH_IMPORT — 批量导入 + +```json +{ + "file_path": "/data/import/model.csv", + "file_type": "CSV", + "options": { + "overwrite": false, + "validate": true, + "notify_user": true + } +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------------- | :------ | :--- | :-------------------------------- | +| `file_path` | string | 是 | 导入文件路径 | +| `file_type` | string | 否 | 文件类型：`CSV`, `JSON`, `XML` | +| `options.overwrite` | boolean | 否 | 是否覆盖已有数据，默认 `false` | +| `options.validate` | boolean | 否 | 是否校验数据，默认 `true` | +| `options.notify_user` | boolean | 否 | 完成后是否通知用户，默认 `true` | + +#### TEST — 测试任务 + +```json +{ + "sleep_duration": 30 +} +``` + +| 字段 | 类型 | 必填 | 说明 | +| :--------------- | :--- | :--- | :---------------------------------------------- | +| `sleep_duration` | int | 否 | 模拟执行耗时（秒），默认 60，最大 3600 | + +### 5.4 任务结果对象（AsyncTaskResult） + +| 字段 | 类型 | 说明 | +| :-------------- | :----- | :--------------------------------------------------- | +| `task_id` | string | 任务 UUID | +| `task_type` | string | 任务类型 | +| `status` | string | 当前状态 | +| `progress` | int | 进度（0-100），仅 `RUNNING` 时返回 | +| `created_at` | int64 | 创建时间戳（Unix 秒） | +| `finished_at` | int64 | 完成时间戳（Unix 秒），仅 `COMPLETED`/`FAILED` 返回 | +| `result` | object | 任务结果，仅 `COMPLETED` 时返回 | +| `error_code` | int | 错误码，仅 `FAILED` 时返回 | +| `error_message` | string | 错误描述，仅 `FAILED` 时返回 | +| `error_detail` | object | 错误详情，仅 `FAILED` 时返回 | + +--- + +## 6. 典型调用流程 + +``` +创建任务 + └─ POST /task/async + └─ 返回 task_id + +轮询状态（建议间隔 2-5 秒） + └─ GET /task/async/{task_id} + ├─ status=SUBMITTED → 继续等待 + ├─ status=RUNNING → 查看 progress + ├─ status=COMPLETED → 读取 result 字段 + └─ status=FAILED → 读取 error_code / error_message + +如需中止（仅 SUBMITTED 状态有效） + └─ POST /task/async/{task_id}/cancel +``` + +--- + +## 7. 队列配置参考 + +| 配置项 | 值 | +| :------------- | :-------------------------- | +| Exchange | `modelrt.tasks.exchange` | +| Queue | `modelrt.tasks.queue` | +| Routing Key | `modelrt.task` | +| 优先级范围 | 0–10（默认 5） | +| 消息 TTL | 24 小时 | +| 最大重试次数 | 3 次 | +| 重试初始延迟 | 1 秒（指数退避，最大 5 分钟）| + +--- + +**文档版本**: 1.1 +**最后更新**: 2026-04-28 +**相关文档**: [异步任务系统设计文档](./async_task_system.md) diff --git a/docs/docs.go b/docs/docs.go index ab3d020..f674f98 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -102,13 +102,12 @@ const docTemplate = `{ "summary": "测量点推荐（搜索框自动补全）", "parameters": [ { - "description": "查询输入参数，例如 'trans' 或 'transformfeeder1_220.'", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/network.MeasurementRecommendRequest" - } + "type": "string", + "example": "\"grid1\"", + "description": "推荐关键词,例如 'grid1' 或 'grid1.'", + "name": "input", + "in": "query", + "required": true } ], "responses": { @@ -176,19 +175,400 @@ const docTemplate = `{ } } } + }, + "/monitors/data/realtime/stream/:clientID": { + "get": { + "description": "根据用户输入的clientID拉取对应的实时数据", + "tags": [ + "RealTime Component Websocket" + ], + "summary": "实时数据拉取 websocket api", + "responses": {} + } + }, + "/monitors/data/subscriptions": { + "post": { + "description": "根据用户输入的组件token,从 modelRT 服务中开始或结束对于量测节点的实时数据的订阅", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "RealTime Component" + ], + "summary": "开始或结束订阅实时数据", + "parameters": [ + { + "description": "量测节点实时数据订阅", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.RealTimeSubRequest" + } + } + ], + "responses": { + "2000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + }, + "3000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.FailureResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + } + } + } + }, + "/task/async": { + "post": { + "description": "创建新的异步任务并返回任务ID，任务将被提交到队列等待处理", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "创建异步任务", + "parameters": [ + { + "description": "任务创建请求", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.AsyncTaskCreateRequest" + } + } + ], + "responses": { + "200": { + "description": "任务创建成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskCreateResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/results": { + "get": { + "description": "根据任务ID列表查询异步任务的状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务结果", + "parameters": [ + { + "type": "string", + "description": "任务ID列表，用逗号分隔", + "name": "task_ids", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResultQueryResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}": { + "get": { + "description": "根据任务ID查询异步任务的详细状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务详情", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}/cancel": { + "post": { + "description": "取消指定ID的异步任务（如果任务尚未开始执行）", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "取消异步任务", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "任务取消成功", + "schema": { + "$ref": "#/definitions/network.SuccessResponse" + } + }, + "400": { + "description": "请求参数错误或任务无法取消", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } } }, "definitions": { + "network.AsyncTaskCreateRequest": { + "type": "object", + "properties": { + "params": { + "description": "required: true", + "type": "object" + }, + "task_type": { + "description": "required: true\nenum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT", + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskCreateResponse": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + } + } + }, + "network.AsyncTaskResult": { + "type": "object", + "properties": { + "created_at": { + "type": "integer", + "example": 1741846200 + }, + "error_code": { + "type": "integer", + "example": 400102 + }, + "error_detail": { + "type": "object" + }, + "error_message": { + "type": "string", + "example": "Component UUID not found" + }, + "finished_at": { + "type": "integer", + "example": 1741846205 + }, + "progress": { + "type": "integer", + "example": 65 + }, + "result": { + "type": "object" + }, + "status": { + "type": "string", + "example": "COMPLETED" + }, + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + }, + "task_type": { + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskResultQueryResponse": { + "type": "object", + "properties": { + "tasks": { + "type": "array", + "items": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + }, + "total": { + "type": "integer", + "example": 3 + } + } + }, "network.FailureResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 500 + "example": 3000 }, "msg": { "type": "string", - "example": "failed to get recommend data from redis" + "example": "process completed with partial failures" }, "payload": { "type": "object" @@ -216,15 +596,10 @@ const docTemplate = `{ " \"I_B_rms\"", "\"I_C_rms\"]" ] - } - } - }, - "network.MeasurementRecommendRequest": { - "type": "object", - "properties": { - "input": { + }, + "recommended_type": { "type": "string", - "example": "trans" + "example": "grid_tag" } } }, @@ -237,21 +612,93 @@ const docTemplate = `{ } } }, + "network.RealTimeMeasurementItem": { + "type": "object", + "properties": { + "interval": { + "type": "string", + "example": "1" + }, + "targets": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "[\"grid1.zone1.station1.ns1.tag1.bay.I11_A_rms\"", + "\"grid1.zone1.station1.ns1.tag1.tag1.bay.I11_B_rms\"]" + ] + } + } + }, + "network.RealTimeSubPayload": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/definitions/network.TargetResult" + } + } + } + }, + "network.RealTimeSubRequest": { + "type": "object", + "properties": { + "action": { + "description": "required: true\nenum: [start, stop]", + "type": "string", + "example": "start" + }, + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "measurements": { + "description": "required: true", + "type": "array", + "items": { + "$ref": "#/definitions/network.RealTimeMeasurementItem" + } + } + } + }, "network.SuccessResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 200 + "example": 2000 }, "msg": { "type": "string", - "example": "success" + "example": "process completed" }, "payload": { "type": "object" } } + }, + "network.TargetResult": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 20000 + }, + "id": { + "type": "string", + "example": "grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms" + }, + "msg": { + "type": "string", + "example": "subscription success" + } + } } } }` diff --git a/docs/swagger.json b/docs/swagger.json index 92f20fa..c6bb033 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -96,13 +96,12 @@ "summary": "测量点推荐（搜索框自动补全）", "parameters": [ { - "description": "查询输入参数，例如 'trans' 或 'transformfeeder1_220.'", - "name": "request", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/network.MeasurementRecommendRequest" - } + "type": "string", + "example": "\"grid1\"", + "description": "推荐关键词,例如 'grid1' 或 'grid1.'", + "name": "input", + "in": "query", + "required": true } ], "responses": { @@ -170,19 +169,400 @@ } } } + }, + "/monitors/data/realtime/stream/:clientID": { + "get": { + "description": "根据用户输入的clientID拉取对应的实时数据", + "tags": [ + "RealTime Component Websocket" + ], + "summary": "实时数据拉取 websocket api", + "responses": {} + } + }, + "/monitors/data/subscriptions": { + "post": { + "description": "根据用户输入的组件token,从 modelRT 服务中开始或结束对于量测节点的实时数据的订阅", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "RealTime Component" + ], + "summary": "开始或结束订阅实时数据", + "parameters": [ + { + "description": "量测节点实时数据订阅", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.RealTimeSubRequest" + } + } + ], + "responses": { + "2000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + }, + "3000": { + "description": "订阅实时数据结果列表", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.FailureResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.RealTimeSubPayload" + } + } + } + ] + } + } + } + } + }, + "/task/async": { + "post": { + "description": "创建新的异步任务并返回任务ID，任务将被提交到队列等待处理", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "创建异步任务", + "parameters": [ + { + "description": "任务创建请求", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/network.AsyncTaskCreateRequest" + } + } + ], + "responses": { + "200": { + "description": "任务创建成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskCreateResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/results": { + "get": { + "description": "根据任务ID列表查询异步任务的状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务结果", + "parameters": [ + { + "type": "string", + "description": "任务ID列表，用逗号分隔", + "name": "task_ids", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResultQueryResponse" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}": { + "get": { + "description": "根据任务ID查询异步任务的详细状态和结果", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "查询异步任务详情", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "查询成功", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/network.SuccessResponse" + }, + { + "type": "object", + "properties": { + "payload": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + } + } + ] + } + }, + "400": { + "description": "请求参数错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } + }, + "/task/async/{task_id}/cancel": { + "post": { + "description": "取消指定ID的异步任务（如果任务尚未开始执行）", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "AsyncTask" + ], + "summary": "取消异步任务", + "parameters": [ + { + "type": "string", + "description": "任务ID", + "name": "task_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "任务取消成功", + "schema": { + "$ref": "#/definitions/network.SuccessResponse" + } + }, + "400": { + "description": "请求参数错误或任务无法取消", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "404": { + "description": "任务不存在", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + }, + "500": { + "description": "服务器内部错误", + "schema": { + "$ref": "#/definitions/network.FailureResponse" + } + } + } + } } }, "definitions": { + "network.AsyncTaskCreateRequest": { + "type": "object", + "properties": { + "params": { + "description": "required: true", + "type": "object" + }, + "task_type": { + "description": "required: true\nenum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT", + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskCreateResponse": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + } + } + }, + "network.AsyncTaskResult": { + "type": "object", + "properties": { + "created_at": { + "type": "integer", + "example": 1741846200 + }, + "error_code": { + "type": "integer", + "example": 400102 + }, + "error_detail": { + "type": "object" + }, + "error_message": { + "type": "string", + "example": "Component UUID not found" + }, + "finished_at": { + "type": "integer", + "example": 1741846205 + }, + "progress": { + "type": "integer", + "example": 65 + }, + "result": { + "type": "object" + }, + "status": { + "type": "string", + "example": "COMPLETED" + }, + "task_id": { + "type": "string", + "example": "123e4567-e89b-12d3-a456-426614174000" + }, + "task_type": { + "type": "string", + "example": "TOPOLOGY_ANALYSIS" + } + } + }, + "network.AsyncTaskResultQueryResponse": { + "type": "object", + "properties": { + "tasks": { + "type": "array", + "items": { + "$ref": "#/definitions/network.AsyncTaskResult" + } + }, + "total": { + "type": "integer", + "example": 3 + } + } + }, "network.FailureResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 500 + "example": 3000 }, "msg": { "type": "string", - "example": "failed to get recommend data from redis" + "example": "process completed with partial failures" }, "payload": { "type": "object" @@ -210,15 +590,10 @@ " \"I_B_rms\"", "\"I_C_rms\"]" ] - } - } - }, - "network.MeasurementRecommendRequest": { - "type": "object", - "properties": { - "input": { + }, + "recommended_type": { "type": "string", - "example": "trans" + "example": "grid_tag" } } }, @@ -231,21 +606,93 @@ } } }, + "network.RealTimeMeasurementItem": { + "type": "object", + "properties": { + "interval": { + "type": "string", + "example": "1" + }, + "targets": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "[\"grid1.zone1.station1.ns1.tag1.bay.I11_A_rms\"", + "\"grid1.zone1.station1.ns1.tag1.tag1.bay.I11_B_rms\"]" + ] + } + } + }, + "network.RealTimeSubPayload": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/definitions/network.TargetResult" + } + } + } + }, + "network.RealTimeSubRequest": { + "type": "object", + "properties": { + "action": { + "description": "required: true\nenum: [start, stop]", + "type": "string", + "example": "start" + }, + "client_id": { + "type": "string", + "example": "5d72f2d9-e33a-4f1b-9c76-88a44b9a953e" + }, + "measurements": { + "description": "required: true", + "type": "array", + "items": { + "$ref": "#/definitions/network.RealTimeMeasurementItem" + } + } + } + }, "network.SuccessResponse": { "type": "object", "properties": { "code": { "type": "integer", - "example": 200 + "example": 2000 }, "msg": { "type": "string", - "example": "success" + "example": "process completed" }, "payload": { "type": "object" } } + }, + "network.TargetResult": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 20000 + }, + "id": { + "type": "string", + "example": "grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms" + }, + "msg": { + "type": "string", + "example": "subscription success" + } + } } } } \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e540f45..598c8d9 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,12 +1,71 @@ basePath: /api/v1 definitions: + network.AsyncTaskCreateRequest: + properties: + params: + description: 'required: true' + type: object + task_type: + description: |- + required: true + enum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT + example: TOPOLOGY_ANALYSIS + type: string + type: object + network.AsyncTaskCreateResponse: + properties: + task_id: + example: 123e4567-e89b-12d3-a456-426614174000 + type: string + type: object + network.AsyncTaskResult: + properties: + created_at: + example: 1741846200 + type: integer + error_code: + example: 400102 + type: integer + error_detail: + type: object + error_message: + example: Component UUID not found + type: string + finished_at: + example: 1741846205 + type: integer + progress: + example: 65 + type: integer + result: + type: object + status: + example: COMPLETED + type: string + task_id: + example: 123e4567-e89b-12d3-a456-426614174000 + type: string + task_type: + example: TOPOLOGY_ANALYSIS + type: string + type: object + network.AsyncTaskResultQueryResponse: + properties: + tasks: + items: + $ref: '#/definitions/network.AsyncTaskResult' + type: array + total: + example: 3 + type: integer + type: object network.FailureResponse: properties: code: - example: 500 + example: 3000 type: integer msg: - example: failed to get recommend data from redis + example: process completed with partial failures type: string payload: type: object @@ -27,11 +86,8 @@ definitions: items: type: string type: array - type: object - network.MeasurementRecommendRequest: - properties: - input: - example: trans + recommended_type: + example: grid_tag type: string type: object network.RealTimeDataPayload: @@ -40,17 +96,69 @@ definitions: description: TODO 增加example tag type: object type: object + network.RealTimeMeasurementItem: + properties: + interval: + example: "1" + type: string + targets: + example: + - '["grid1.zone1.station1.ns1.tag1.bay.I11_A_rms"' + - '"grid1.zone1.station1.ns1.tag1.tag1.bay.I11_B_rms"]' + items: + type: string + type: array + type: object + network.RealTimeSubPayload: + properties: + client_id: + example: 5d72f2d9-e33a-4f1b-9c76-88a44b9a953e + type: string + targets: + items: + $ref: '#/definitions/network.TargetResult' + type: array + type: object + network.RealTimeSubRequest: + properties: + action: + description: |- + required: true + enum: [start, stop] + example: start + type: string + client_id: + example: 5d72f2d9-e33a-4f1b-9c76-88a44b9a953e + type: string + measurements: + description: 'required: true' + items: + $ref: '#/definitions/network.RealTimeMeasurementItem' + type: array + type: object network.SuccessResponse: properties: code: - example: 200 + example: 2000 type: integer msg: - example: success + example: process completed type: string payload: type: object type: object + network.TargetResult: + properties: + code: + example: 20000 + type: integer + id: + example: grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms + type: string + msg: + example: subscription success + type: string + type: object host: localhost:8080 info: contact: @@ -110,12 +218,12 @@ paths: - application/json description: 根据用户输入的字符串，从 Redis 中查询可能的测量点或结构路径，并提供推荐列表。 parameters: - - description: 查询输入参数，例如 'trans' 或 'transformfeeder1_220.' - in: body - name: request + - description: 推荐关键词,例如 'grid1' 或 'grid1.' + example: '"grid1"' + in: query + name: input required: true - schema: - $ref: '#/definitions/network.MeasurementRecommendRequest' + type: string produces: - application/json responses: @@ -160,4 +268,187 @@ paths: summary: load circuit diagram info tags: - load circuit_diagram + /monitors/data/realtime/stream/:clientID: + get: + description: 根据用户输入的clientID拉取对应的实时数据 + responses: {} + summary: 实时数据拉取 websocket api + tags: + - RealTime Component Websocket + /monitors/data/subscriptions: + post: + consumes: + - application/json + description: 根据用户输入的组件token,从 modelRT 服务中开始或结束对于量测节点的实时数据的订阅 + parameters: + - description: 量测节点实时数据订阅 + in: body + name: request + required: true + schema: + $ref: '#/definitions/network.RealTimeSubRequest' + produces: + - application/json + responses: + "2000": + description: 订阅实时数据结果列表 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.RealTimeSubPayload' + type: object + "3000": + description: 订阅实时数据结果列表 + schema: + allOf: + - $ref: '#/definitions/network.FailureResponse' + - properties: + payload: + $ref: '#/definitions/network.RealTimeSubPayload' + type: object + summary: 开始或结束订阅实时数据 + tags: + - RealTime Component + /task/async: + post: + consumes: + - application/json + description: 创建新的异步任务并返回任务ID，任务将被提交到队列等待处理 + parameters: + - description: 任务创建请求 + in: body + name: request + required: true + schema: + $ref: '#/definitions/network.AsyncTaskCreateRequest' + produces: + - application/json + responses: + "200": + description: 任务创建成功 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.AsyncTaskCreateResponse' + type: object + "400": + description: 请求参数错误 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 创建异步任务 + tags: + - AsyncTask + /task/async/{task_id}: + get: + consumes: + - application/json + description: 根据任务ID查询异步任务的详细状态和结果 + parameters: + - description: 任务ID + in: path + name: task_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: 查询成功 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.AsyncTaskResult' + type: object + "400": + description: 请求参数错误 + schema: + $ref: '#/definitions/network.FailureResponse' + "404": + description: 任务不存在 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 查询异步任务详情 + tags: + - AsyncTask + /task/async/{task_id}/cancel: + post: + consumes: + - application/json + description: 取消指定ID的异步任务（如果任务尚未开始执行） + parameters: + - description: 任务ID + in: path + name: task_id + required: true + type: string + produces: + - application/json + responses: + "200": + description: 任务取消成功 + schema: + $ref: '#/definitions/network.SuccessResponse' + "400": + description: 请求参数错误或任务无法取消 + schema: + $ref: '#/definitions/network.FailureResponse' + "404": + description: 任务不存在 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 取消异步任务 + tags: + - AsyncTask + /task/async/results: + get: + consumes: + - application/json + description: 根据任务ID列表查询异步任务的状态和结果 + parameters: + - description: 任务ID列表，用逗号分隔 + in: query + name: task_ids + required: true + type: string + produces: + - application/json + responses: + "200": + description: 查询成功 + schema: + allOf: + - $ref: '#/definitions/network.SuccessResponse' + - properties: + payload: + $ref: '#/definitions/network.AsyncTaskResultQueryResponse' + type: object + "400": + description: 请求参数错误 + schema: + $ref: '#/definitions/network.FailureResponse' + "500": + description: 服务器内部错误 + schema: + $ref: '#/definitions/network.FailureResponse' + summary: 查询异步任务结果 + tags: + - AsyncTask swagger: "2.0" diff --git a/go.mod b/go.mod index 3695aef..441ab18 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,33 @@ module modelRT -go 1.24 +go 1.26.3 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/RediSearch/redisearch-go/v2 v2.1.1 github.com/bitly/go-simplejson v0.5.1 - github.com/gin-gonic/gin v1.10.0 + github.com/gin-contrib/cors v1.7.6 + github.com/gin-gonic/gin v1.10.1 github.com/gofrs/uuid v4.4.0+incompatible github.com/gomodule/redigo v1.8.9 github.com/gorilla/websocket v1.5.3 github.com/json-iterator/go v1.1.12 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/panjf2000/ants/v2 v2.10.0 + github.com/rabbitmq/amqp091-go v1.10.0 github.com/redis/go-redis/v9 v9.7.3 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.11.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.4 + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/zap v1.27.0 - golang.org/x/sys v0.28.0 + golang.org/x/sync v0.20.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 gorm.io/gorm v1.25.12 @@ -29,25 +36,29 @@ require ( require ( github.com/BurntSushi/toml v1.4.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/bytedance/sonic v1.12.5 // indirect - github.com/bytedance/sonic/loader v0.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/bytedance/sonic v1.13.3 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.23.0 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -56,7 +67,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -64,7 +75,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -74,16 +85,23 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.12 // indirect + github.com/ugorji/go/codec v1.3.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/arch v0.12.0 // indirect - golang.org/x/crypto v0.30.0 // indirect + golang.org/x/arch v0.18.0 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.32.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/tools v0.28.0 // indirect - google.golang.org/protobuf v1.35.2 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect + golang.org/x/tools v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 33b982e..19c0d18 100644 --- a/go.sum +++ b/go.sum @@ -12,16 +12,17 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.12.5 h1:hoZxY8uW+mT+OpkcUWw4k0fDINtOcVavEsGfzwzFU/w= -github.com/bytedance/sonic v1.12.5/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= +github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= -github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -33,14 +34,21 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= -github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gin-contrib/cors v1.7.6 h1:3gQ8GMzs1Ylpf70y8bMw4fVpycXIeX1ZemuSQIsnQQY= +github.com/gin-contrib/cors v1.7.6/go.mod h1:Ulcl+xN4jel9t1Ry8vqph23a60FwH9xVLd+3ykmTjOk= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -55,21 +63,27 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= -github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -90,8 +104,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -116,15 +130,17 @@ github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -148,8 +164,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= @@ -160,37 +176,57 @@ github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= +github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= -golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= +golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -198,8 +234,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -207,16 +243,24 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/handler/alert_event_query.go b/handler/alert_event_query.go index 58e1613..b49d954 100644 --- a/handler/alert_event_query.go +++ b/handler/alert_event_query.go @@ -5,10 +5,10 @@ import ( "net/http" "strconv" - "modelRT/alert" "modelRT/constants" "modelRT/logger" "modelRT/network" + "modelRT/real-time-data/alert" "github.com/gin-gonic/gin" ) diff --git a/handler/anchor_point_replace.go b/handler/anchor_point_replace.go index 877c877..9bdf0ab 100644 --- a/handler/anchor_point_replace.go +++ b/handler/anchor_point_replace.go @@ -68,7 +68,7 @@ func ComponentAnchorReplaceHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": request.UUID, }, } diff --git a/handler/async_task_cancel_handler.go b/handler/async_task_cancel_handler.go new file mode 100644 index 0000000..40fa480 --- /dev/null +++ b/handler/async_task_cancel_handler.go @@ -0,0 +1,92 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "time" + + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/mq" + "modelRT/mq/event" + "modelRT/orm" + + "github.com/gin-gonic/gin" + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// AsyncTaskCancelHandler handles cancellation of an async task +// @Summary 取消异步任务 +// @Description 取消指定ID的异步任务（如果任务尚未开始执行） +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} network.SuccessResponse "任务取消成功" +// @Failure 200 {object} network.FailureResponse "请求参数错误或任务无法取消" +// @Router /task/async/{task_id}/cancel [post] +func AsyncTaskCancelHandler(c *gin.Context) { + ctx := c.Request.Context() + + taskIDStr := c.Param("task_id") + if taskIDStr == "" { + logger.Error(ctx, "task_id parameter is required") + renderRespFailure(c, constants.RespCodeInvalidParams, "task_id parameter is required", nil) + return + } + + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task ID format", nil) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) + return + } + + asyncTask, err := database.GetAsyncTaskByID(ctx, pgClient, taskID) + if err != nil { + if err == gorm.ErrRecordNotFound { + logger.Error(ctx, "async task not found", "task_id", taskID) + renderRespFailure(c, constants.RespCodeInvalidParams, "task not found", nil) + return + } + logger.Error(ctx, "failed to query async task from database", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task", nil) + return + } + + if asyncTask.Status != orm.AsyncTaskStatusSubmitted { + logger.Error(ctx, "task cannot be cancelled", "task_id", taskID, "status", asyncTask.Status) + renderRespFailure(c, constants.RespCodeInvalidParams, "task cannot be cancelled, already running or completed", nil) + return + } + + timestamp := time.Now().Unix() + err = database.FailAsyncTask(ctx, pgClient, taskID, timestamp) + if err != nil { + logger.Error(ctx, "failed to cancel async task", "task_id", taskID, "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to cancel task", nil) + return + } + + if record, evtErr := event.NewTaskCancelledMessage(taskID.String(), string(asyncTask.TaskType)); evtErr == nil { + mq.TryEmitMessage(ctx, record) + } + + err = database.UpdateAsyncTaskResultWithError(ctx, pgClient, taskID, 40009, "task cancelled by user", orm.JSONMap{ + "cancelled_at": timestamp, + "cancelled_by": "user", + }) + if err != nil { + logger.Error(ctx, "failed to update task result with cancellation error", "task_id", taskID, "error", err) + } + + renderRespSuccess(c, constants.RespCodeSuccess, "task cancelled successfully", nil) +} diff --git a/handler/async_task_create_handler.go b/handler/async_task_create_handler.go new file mode 100644 index 0000000..d1ff912 --- /dev/null +++ b/handler/async_task_create_handler.go @@ -0,0 +1,163 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/mq" + "modelRT/mq/event" + "modelRT/network" + "modelRT/orm" + "modelRT/task" + + "github.com/gin-gonic/gin" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +// AsyncTaskCreateHandler handles creation of asynchronous tasks +// @Summary 创建异步任务 +// @Description 创建新的异步任务并返回任务ID，任务将被提交到队列等待处理 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param request body network.AsyncTaskCreateRequest true "任务创建请求" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskCreateResponse} "任务创建成功" +// @Failure 400 {object} network.FailureResponse "请求参数错误" +// @Failure 500 {object} network.FailureResponse "服务器内部错误" +// @Router /task/async [post] +func AsyncTaskCreateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskCreateRequest + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "unmarshal async task create request failed", "error", err) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid request parameters", nil) + return + } + + // validate task type + if !orm.IsValidAsyncTaskType(request.TaskType) { + logger.Error(ctx, "check task type invalid", "task_type", request.TaskType) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task type", nil) + return + } + + // validate task parameters based on task type + if !validateTaskParams(request.TaskType, request.Params) { + logger.Error(ctx, "check task parameters invalid", "task_type", request.TaskType, "params", request.Params) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task parameters", nil) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) + return + } + + // create task in database + taskType := orm.AsyncTaskType(request.TaskType) + params := orm.JSONMap(request.Params) + + asyncTask, err := database.CreateAsyncTask(ctx, pgClient, taskType, params) + if err != nil { + logger.Error(ctx, "create async task in database failed", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to create task", nil) + return + } + + // enqueue task to channel for async publishing to RabbitMQ + msg := task.NewTaskQueueMessageWithPriority(asyncTask.TaskID, task.TaskType(request.TaskType), 5) + // propagate the current OTel span context so the async chain stays on the same trace + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + msg.TraceCarrier = carrier + msg.Params = request.Params + task.TaskMsgChan <- msg + logger.Info(ctx, "task enqueued to channel", "task_id", asyncTask.TaskID, "queue", constants.TaskQueueName) + + if record, err := event.NewTaskSubmittedMessage(asyncTask.TaskID.String(), request.TaskType, 5); err == nil { + mq.TryEmitMessage(ctx, record) + } + + logger.Info(ctx, "async task created success", "task_id", asyncTask.TaskID, "task_type", request.TaskType) + + // return success response + payload := genAsyncTaskCreatePayload(asyncTask.TaskID.String()) + renderRespSuccess(c, constants.RespCodeSuccess, "task created successfully", payload) +} + +func validateTaskParams(taskType string, params map[string]any) bool { + switch taskType { + case string(orm.AsyncTaskTypeTopologyAnalysis): + return validateTopologyAnalysisParams(params) + case string(orm.AsyncTaskTypePerformanceAnalysis): + return validatePerformanceAnalysisParams(params) + case string(orm.AsyncTaskTypeEventAnalysis): + return validateEventAnalysisParams(params) + case string(orm.AsyncTaskTypeBatchImport): + return validateBatchImportParams(params) + case string(orm.AsyncTaskTypeTest): + return validateTestTaskParams(params) + default: + return false + } +} + +func validateTopologyAnalysisParams(params map[string]any) bool { + if v, ok := params["start_component_uuid"]; !ok || v == "" { + return false + } + if v, ok := params["end_component_uuid"]; !ok || v == "" { + return false + } + // check_in_service is optional; validate type when present + if v, exists := params["check_in_service"]; exists { + if _, isBool := v.(bool); !isBool { + return false + } + } + return true +} + +func validatePerformanceAnalysisParams(params map[string]any) bool { + // Check required parameters for performance analysis + if componentIDs, ok := params["component_ids"]; !ok { + return false + } else if ids, isSlice := componentIDs.([]any); !isSlice || len(ids) == 0 { + return false + } + return true +} + +func validateEventAnalysisParams(params map[string]any) bool { + // Check required parameters for event analysis + if eventType, ok := params["event_type"]; !ok || eventType == "" { + return false + } + return true +} + +func validateBatchImportParams(params map[string]any) bool { + // Check required parameters for batch import + if filePath, ok := params["file_path"]; !ok || filePath == "" { + return false + } + return true +} + +func validateTestTaskParams(params map[string]any) bool { + // Test task has optional parameters, all are valid + // sleep_duration defaults to 60 seconds if not provided + return true +} + +func genAsyncTaskCreatePayload(taskID string) map[string]any { + payload := map[string]any{ + "task_id": taskID, + } + return payload +} diff --git a/handler/async_task_progress_update_handler.go b/handler/async_task_progress_update_handler.go new file mode 100644 index 0000000..08276ce --- /dev/null +++ b/handler/async_task_progress_update_handler.go @@ -0,0 +1,39 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/network" + + "github.com/gin-gonic/gin" +) + +// AsyncTaskProgressUpdateHandler handles updating task progress (internal use, not exposed via API) +func AsyncTaskProgressUpdateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskProgressUpdate + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task progress update request", "error", err) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid request parameters", nil) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) + return + } + + err := database.UpdateAsyncTaskProgress(ctx, pgClient, request.TaskID, request.Progress) + if err != nil { + logger.Error(ctx, "failed to update async task progress", "task_id", request.TaskID, "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to update task progress", nil) + return + } + + renderRespSuccess(c, constants.RespCodeSuccess, "task progress updated successfully", nil) +} diff --git a/handler/async_task_result_detail_handler.go b/handler/async_task_result_detail_handler.go new file mode 100644 index 0000000..2d4174a --- /dev/null +++ b/handler/async_task_result_detail_handler.go @@ -0,0 +1,93 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/network" + + "github.com/gin-gonic/gin" + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// AsyncTaskResultDetailHandler handles detailed query of a single async task result +// @Summary 查询异步任务详情 +// @Description 根据任务ID查询异步任务的详细状态和结果 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResult} "查询成功" +// @Failure 200 {object} network.FailureResponse "请求参数错误" +// @Router /task/async/{task_id} [get] +func AsyncTaskResultDetailHandler(c *gin.Context) { + ctx := c.Request.Context() + + taskIDStr := c.Param("task_id") + if taskIDStr == "" { + logger.Error(ctx, "task_id parameter is required") + renderRespFailure(c, constants.RespCodeInvalidParams, "task_id parameter is required", nil) + return + } + + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task ID format", nil) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) + return + } + + asyncTask, err := database.GetAsyncTaskByID(ctx, pgClient, taskID) + if err != nil { + if err == gorm.ErrRecordNotFound { + logger.Error(ctx, "async task not found", "task_id", taskID) + renderRespFailure(c, constants.RespCodeInvalidParams, "task not found", nil) + return + } + logger.Error(ctx, "failed to query async task from database", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task", nil) + return + } + + taskResult, err := database.GetAsyncTaskResult(ctx, pgClient, taskID) + if err != nil { + logger.Error(ctx, "failed to query async task result from database", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task result", nil) + return + } + + responseTask := network.AsyncTaskResult{ + TaskID: asyncTask.TaskID, + TaskType: string(asyncTask.TaskType), + Status: string(asyncTask.Status), + CreatedAt: asyncTask.CreatedAt, + FinishedAt: asyncTask.FinishedAt, + Progress: asyncTask.Progress, + } + + if taskResult != nil { + if taskResult.Result != nil { + responseTask.Result = map[string]any(taskResult.Result) + } + if taskResult.ErrorCode != nil { + responseTask.ErrorCode = taskResult.ErrorCode + } + if taskResult.ErrorMessage != nil { + responseTask.ErrorMessage = taskResult.ErrorMessage + } + if taskResult.ErrorDetail != nil { + responseTask.ErrorDetail = map[string]any(taskResult.ErrorDetail) + } + } + + renderRespSuccess(c, constants.RespCodeSuccess, "query completed", responseTask) +} diff --git a/handler/async_task_result_query_handler.go b/handler/async_task_result_query_handler.go new file mode 100644 index 0000000..e9b08ae --- /dev/null +++ b/handler/async_task_result_query_handler.go @@ -0,0 +1,151 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "strings" + + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/network" + "modelRT/orm" + + "github.com/gin-gonic/gin" + "github.com/gofrs/uuid" +) + +// AsyncTaskResultQueryHandler handles querying of asynchronous task results +// @Summary 查询异步任务结果 +// @Description 根据任务ID列表查询异步任务的状态和结果 +// @Tags AsyncTask +// @Accept json +// @Produce json +// @Param task_ids query string true "任务ID列表，用逗号分隔" +// @Success 200 {object} network.SuccessResponse{payload=network.AsyncTaskResultQueryResponse} "查询成功" +// @Failure 200 {object} network.FailureResponse "请求参数错误" +// @Router /task/async/results [get] +func AsyncTaskResultQueryHandler(c *gin.Context) { + ctx := c.Request.Context() + + taskIDsParam := c.Query("task_ids") + if taskIDsParam == "" { + logger.Error(ctx, "task_ids parameter is required") + renderRespFailure(c, constants.RespCodeInvalidParams, "task_ids parameter is required", nil) + return + } + + var taskIDs []uuid.UUID + taskIDStrs := splitCommaSeparated(taskIDsParam) + for _, taskIDStr := range taskIDStrs { + taskID, err := uuid.FromString(taskIDStr) + if err != nil { + logger.Error(ctx, "invalid task ID format", "task_id", taskIDStr, "error", err) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task ID format", nil) + return + } + taskIDs = append(taskIDs, taskID) + } + + if len(taskIDs) == 0 { + logger.Error(ctx, "no valid task IDs provided") + renderRespFailure(c, constants.RespCodeInvalidParams, "no valid task IDs provided", nil) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) + return + } + + asyncTasks, err := database.GetAsyncTasksByIDs(ctx, pgClient, taskIDs) + if err != nil { + logger.Error(ctx, "failed to query async tasks from database", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to query tasks", nil) + return + } + + taskResults, err := database.GetAsyncTaskResults(ctx, pgClient, taskIDs) + if err != nil { + logger.Error(ctx, "failed to query async task results from database", "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to query task results", nil) + return + } + + taskResultMap := make(map[uuid.UUID]orm.AsyncTaskResult) + for _, result := range taskResults { + taskResultMap[result.TaskID] = result + } + + var responseTasks []network.AsyncTaskResult + for _, asyncTask := range asyncTasks { + taskResult := network.AsyncTaskResult{ + TaskID: asyncTask.TaskID, + TaskType: string(asyncTask.TaskType), + Status: string(asyncTask.Status), + CreatedAt: asyncTask.CreatedAt, + FinishedAt: asyncTask.FinishedAt, + Progress: asyncTask.Progress, + } + + if result, exists := taskResultMap[asyncTask.TaskID]; exists { + if result.Result != nil { + taskResult.Result = map[string]any(result.Result) + } + if result.ErrorCode != nil { + taskResult.ErrorCode = result.ErrorCode + } + if result.ErrorMessage != nil { + taskResult.ErrorMessage = result.ErrorMessage + } + if result.ErrorDetail != nil { + taskResult.ErrorDetail = map[string]any(result.ErrorDetail) + } + } + + responseTasks = append(responseTasks, taskResult) + } + + renderRespSuccess(c, constants.RespCodeSuccess, "query completed", network.AsyncTaskResultQueryResponse{ + Total: len(responseTasks), + Tasks: responseTasks, + }) +} + +func splitCommaSeparated(s string) []string { + var result []string + var current strings.Builder + inQuotes := false + escape := false + + for _, ch := range s { + if escape { + current.WriteRune(ch) + escape = false + continue + } + + switch ch { + case '\\': + escape = true + case '"': + inQuotes = !inQuotes + case ',': + if !inQuotes { + result = append(result, strings.TrimSpace(current.String())) + current.Reset() + } else { + current.WriteRune(ch) + } + default: + current.WriteRune(ch) + } + } + + if current.Len() > 0 { + result = append(result, strings.TrimSpace(current.String())) + } + + return result +} diff --git a/handler/async_task_status_update_handler.go b/handler/async_task_status_update_handler.go new file mode 100644 index 0000000..daf3c72 --- /dev/null +++ b/handler/async_task_status_update_handler.go @@ -0,0 +1,66 @@ +// Package handler provides HTTP handlers for various endpoints. +package handler + +import ( + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/network" + "modelRT/orm" + + "github.com/gin-gonic/gin" +) + +// AsyncTaskStatusUpdateHandler handles updating task status (internal use, not exposed via API) +func AsyncTaskStatusUpdateHandler(c *gin.Context) { + ctx := c.Request.Context() + var request network.AsyncTaskStatusUpdate + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "failed to unmarshal async task status update request", "error", err) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid request parameters", nil) + return + } + + validStatus := map[string]bool{ + string(orm.AsyncTaskStatusSubmitted): true, + string(orm.AsyncTaskStatusRunning): true, + string(orm.AsyncTaskStatusCompleted): true, + string(orm.AsyncTaskStatusFailed): true, + } + + if !validStatus[request.Status] { + logger.Error(ctx, "invalid task status", "status", request.Status) + renderRespFailure(c, constants.RespCodeInvalidParams, "invalid task status", nil) + return + } + + pgClient := database.GetPostgresDBClient() + if pgClient == nil { + logger.Error(ctx, "database connection not found in context") + renderRespFailure(c, constants.RespCodeServerError, "database connection error", nil) + return + } + + status := orm.AsyncTaskStatus(request.Status) + err := database.UpdateAsyncTaskStatus(ctx, pgClient, request.TaskID, status) + if err != nil { + logger.Error(ctx, "failed to update async task status", "task_id", request.TaskID, "status", request.Status, "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to update task status", nil) + return + } + + if request.Status == string(orm.AsyncTaskStatusCompleted) { + err = database.CompleteAsyncTask(ctx, pgClient, request.TaskID, request.Timestamp) + } else if request.Status == string(orm.AsyncTaskStatusFailed) { + err = database.FailAsyncTask(ctx, pgClient, request.TaskID, request.Timestamp) + } + + if err != nil { + logger.Error(ctx, "failed to update async task completion timestamp", "task_id", request.TaskID, "error", err) + renderRespFailure(c, constants.RespCodeServerError, "failed to update task completion timestamp", nil) + return + } + + renderRespSuccess(c, constants.RespCodeSuccess, "task status updated successfully", nil) +} diff --git a/handler/attr_delete.go b/handler/attr_delete.go index b78e46c..783c01f 100644 --- a/handler/attr_delete.go +++ b/handler/attr_delete.go @@ -3,7 +3,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/diagram" "modelRT/logger" "modelRT/network" @@ -16,7 +16,7 @@ func AttrDeleteHandler(c *gin.Context) { var request network.AttrDeleteRequest clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ @@ -41,7 +41,7 @@ func AttrDeleteHandler(c *gin.Context) { c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{"attr_token": request.AttrToken}, + Payload: map[string]any{"attr_token": request.AttrToken}, }) return } @@ -49,7 +49,7 @@ func AttrDeleteHandler(c *gin.Context) { c.JSON(http.StatusOK, network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "attr_token": request.AttrToken, }, }) diff --git a/handler/attr_load.go b/handler/attr_load.go index 3a50be6..3591910 100644 --- a/handler/attr_load.go +++ b/handler/attr_load.go @@ -3,7 +3,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/database" "modelRT/logger" "modelRT/network" @@ -17,7 +17,7 @@ func AttrGetHandler(c *gin.Context) { clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ @@ -46,7 +46,7 @@ func AttrGetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{"attr_token": request.AttrToken}, + Payload: map[string]any{"attr_token": request.AttrToken}, }) return } @@ -59,7 +59,7 @@ func AttrGetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "attr_token": request.AttrToken, "attr_value": attrValue, }, diff --git a/handler/attr_update.go b/handler/attr_update.go index 589164e..226631f 100644 --- a/handler/attr_update.go +++ b/handler/attr_update.go @@ -3,7 +3,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/diagram" "modelRT/logger" "modelRT/network" @@ -17,7 +17,7 @@ func AttrSetHandler(c *gin.Context) { clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ @@ -43,7 +43,7 @@ func AttrSetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{"attr_token": request.AttrToken}, + Payload: map[string]any{"attr_token": request.AttrToken}, }) return } @@ -51,7 +51,7 @@ func AttrSetHandler(c *gin.Context) { c.JSON(http.StatusOK, network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "attr_token": request.AttrToken, }, }) diff --git a/handler/circuit_diagram_create.go b/handler/circuit_diagram_create.go index 536151d..854e0c8 100644 --- a/handler/circuit_diagram_create.go +++ b/handler/circuit_diagram_create.go @@ -37,7 +37,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } @@ -65,7 +65,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicLink, }, } @@ -89,7 +89,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_infos": topologicCreateInfos, }, } @@ -111,7 +111,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "component_infos": request.ComponentInfos, }, } @@ -130,7 +130,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": info.UUID, "component_params": info.Params, }, @@ -152,7 +152,7 @@ func CircuitDiagramCreateHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } diff --git a/handler/circuit_diagram_delete.go b/handler/circuit_diagram_delete.go index a691679..2025ad0 100644 --- a/handler/circuit_diagram_delete.go +++ b/handler/circuit_diagram_delete.go @@ -42,7 +42,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } @@ -70,7 +70,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicLink, }, } @@ -95,7 +95,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicDelInfo, }, } @@ -112,7 +112,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicDelInfo, }, } @@ -138,7 +138,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentInfo.UUID, }, } @@ -162,7 +162,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentInfo.UUID, }, } @@ -184,7 +184,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentInfo.UUID, }, } @@ -205,7 +205,7 @@ func CircuitDiagramDeleteHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } diff --git a/handler/circuit_diagram_load.go b/handler/circuit_diagram_load.go index 8f8c71c..7ad390b 100644 --- a/handler/circuit_diagram_load.go +++ b/handler/circuit_diagram_load.go @@ -33,7 +33,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": pageID, }, } @@ -48,14 +48,14 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": pageID, }, } c.JSON(http.StatusOK, resp) return } - payload := make(map[string]interface{}) + payload := make(map[string]any) payload["root_vertex"] = topologicInfo.RootVertex payload["topologic"] = topologicInfo.VerticeLinks @@ -69,7 +69,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentUUID, }, } @@ -84,7 +84,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": componentUUID, }, } @@ -103,7 +103,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": topologicInfo.RootVertex, }, } @@ -118,7 +118,7 @@ func CircuitDiagramLoadHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": rootVertexUUID, }, } diff --git a/handler/circuit_diagram_update.go b/handler/circuit_diagram_update.go index d0181e1..33bb46b 100644 --- a/handler/circuit_diagram_update.go +++ b/handler/circuit_diagram_update.go @@ -35,7 +35,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } @@ -52,7 +52,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicLink, }, } @@ -75,7 +75,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicChangeInfo, }, } @@ -92,7 +92,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "topologic_info": topologicChangeInfo, }, } @@ -109,7 +109,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, "component_info": request.ComponentInfos, }, @@ -129,7 +129,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.FailureResponse{ Code: http.StatusBadRequest, Msg: err.Error(), - Payload: map[string]interface{}{ + Payload: map[string]any{ "uuid": info.UUID, "component_params": info.Params, }, @@ -152,7 +152,7 @@ func CircuitDiagramUpdateHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "page_id": request.PageID, }, } diff --git a/handler/diagram_node_link.go b/handler/diagram_node_link.go index 80850e7..8f484a5 100644 --- a/handler/diagram_node_link.go +++ b/handler/diagram_node_link.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "modelRT/common" "modelRT/constants" "modelRT/database" "modelRT/diagram" @@ -43,7 +44,7 @@ func DiagramNodeLinkHandler(c *gin.Context) { var request network.DiagramNodeLinkRequest clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, @@ -167,7 +168,7 @@ func processLinkSetData(ctx context.Context, action string, level int, prevLinkS err2 = prevLinkSet.SREM(prevMember) } default: - err := constants.ErrUnsupportedLinkAction + err := common.ErrUnsupportedLinkAction logger.Error(ctx, "unsupport diagram node link process action", "action", action, "error", err) return err } diff --git a/handler/helper.go b/handler/helper.go index 6589409..532d93b 100644 --- a/handler/helper.go +++ b/handler/helper.go @@ -30,3 +30,14 @@ func renderRespSuccess(c *gin.Context, code int, msg string, payload any) { } c.JSON(http.StatusOK, resp) } + +func renderWSRespFailure(c *gin.Context, code int, msg string, payload any) { + resp := network.WSResponse{ + Code: code, + Msg: msg, + } + if payload != nil { + resp.Payload = payload + } + c.JSON(http.StatusOK, resp) +} diff --git a/handler/history_data_query.go b/handler/history_data_query.go index 294f4b9..f9df2a2 100644 --- a/handler/history_data_query.go +++ b/handler/history_data_query.go @@ -6,10 +6,10 @@ import ( "net/http" "strconv" - "modelRT/alert" "modelRT/constants" "modelRT/logger" "modelRT/network" + "modelRT/real-time-data/alert" "github.com/gin-gonic/gin" ) @@ -50,7 +50,7 @@ func QueryHistoryDataHandler(c *gin.Context) { resp := network.SuccessResponse{ Code: http.StatusOK, Msg: "success", - Payload: map[string]interface{}{ + Payload: map[string]any{ "events": events, }, } diff --git a/handler/measurement_load.go b/handler/measurement_load.go index 065b9f5..26dbbed 100644 --- a/handler/measurement_load.go +++ b/handler/measurement_load.go @@ -4,7 +4,7 @@ package handler import ( "net/http" - "modelRT/constants" + "modelRT/common" "modelRT/database" "modelRT/diagram" "modelRT/logger" @@ -19,7 +19,7 @@ func MeasurementGetHandler(c *gin.Context) { clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ diff --git a/handler/mesurement_link.go b/handler/mesurement_link.go index b45e8bb..1a878f6 100644 --- a/handler/mesurement_link.go +++ b/handler/mesurement_link.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" + "modelRT/common" "modelRT/constants" "modelRT/database" "modelRT/diagram" @@ -20,7 +21,7 @@ func MeasurementLinkHandler(c *gin.Context) { var request network.MeasurementLinkRequest clientToken := c.GetString("client_token") if clientToken == "" { - err := constants.ErrGetClientToken + err := common.ErrGetClientToken logger.Error(c, "failed to get client token from context", "error", err) c.JSON(http.StatusOK, network.FailureResponse{ Code: http.StatusBadRequest, @@ -93,7 +94,7 @@ func MeasurementLinkHandler(c *gin.Context) { logger.Error(c, "del measurement link process operation failed", "measurement_id", measurementID, "action", action, "error", err) } default: - err = constants.ErrUnsupportedLinkAction + err = common.ErrUnsupportedLinkAction logger.Error(c, "unsupport measurement link process action", "measurement_id", measurementID, "action", action, "error", err) } diff --git a/handler/real_time_data_pull.go b/handler/real_time_data_pull.go index c190ff1..986ed11 100644 --- a/handler/real_time_data_pull.go +++ b/handler/real_time_data_pull.go @@ -39,20 +39,14 @@ func PullRealTimeDataHandler(c *gin.Context) { if clientID == "" { err := fmt.Errorf("clientID is missing from the path") logger.Error(c, "query clientID from path failed", "error", err, "url", c.Request.RequestURI) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + renderWSRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil) return } conn, err := pullUpgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { - logger.Error(c, "upgrade http protocol to websocket protocal failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + logger.Error(c, "upgrade http protocol to websocket protocol failed", "error", err) + renderWSRespFailure(c, constants.RespCodeServerError, err.Error(), nil) return } defer conn.Close() @@ -60,9 +54,18 @@ func PullRealTimeDataHandler(c *gin.Context) { ctx, cancel := context.WithCancel(c.Request.Context()) defer cancel() + conn.SetCloseHandler(func(code int, text string) error { + logger.Info(c.Request.Context(), "websocket processor shutdown trigger", + "clientID", clientID, "code", code, "reason", text) + + // call cancel to notify other goroutines to stop working + cancel() + return nil + }) + // TODO[BACKPRESSURE-ISSUE] 先期使用固定大容量对扇入模型进行定义 #1 fanInChan := make(chan network.RealTimePullTarget, constants.FanInChanMaxSize) - sendChan := make(chan []network.RealTimePullTarget, constants.SendChanBufferSize) + sendChan := make(chan network.WSResponse, constants.SendChanBufferSize) go processTargetPolling(ctx, globalSubState, clientID, fanInChan, sendChan) go readClientMessages(ctx, conn, clientID, cancel) @@ -79,52 +82,33 @@ func PullRealTimeDataHandler(c *gin.Context) { select { case targetData, ok := <-fanInChan: if !ok { - logger.Error(ctx, "fanInChan closed unexpectedly", "client_id", clientID) + sendChan <- network.WSResponse{ + Code: constants.RespCodeServerError, + Msg: "abnormal shutdown of data fan-in channel", + } return } + buffer = append(buffer, targetData) - if len(buffer) >= bufferMaxSize { - // buffer is full, send immediately - select { - case sendChan <- buffer: - default: - logger.Warn(ctx, "sendChan is full, dropping aggregated data batch (buffer is full)", "client_id", clientID) - } - - // reset buffer - buffer = make([]network.RealTimePullTarget, 0, bufferMaxSize) - // reset the ticker to prevent it from triggering immediately after the ticker is sent + flushBuffer(ctx, &buffer, sendChan, clientID, "buffer_full") ticker.Reset(sendMaxInterval) } case <-ticker.C: if len(buffer) > 0 { - // when the ticker is triggered, all data in the send buffer is sent - select { - case sendChan <- buffer: - default: - logger.Warn(ctx, "sendChan is full, dropping aggregated data batch (ticker is triggered)", "client_id", clientID) - } - - // reset buffer - buffer = make([]network.RealTimePullTarget, 0, bufferMaxSize) + flushBuffer(ctx, &buffer, sendChan, clientID, "ticker_timeout") } case <-ctx.Done(): - // send the last remaining data + // last refresh before exiting if len(buffer) > 0 { - select { - case sendChan <- buffer: - default: - logger.Warn(ctx, "sendChan is full, cannot send last remaining data during shutdown.", "client_id", clientID) - } + flushBuffer(ctx, &buffer, sendChan, clientID, "shutdown") } - logger.Info(ctx, "pullRealTimeDataHandler exiting as context is done.", "client_id", clientID) return } } } -// readClientMessages 负责持续监听客户端发送的消息（例如 Ping/Pong, Close Frame, 或控制命令） +// readClientMessages define func to responsible for continuously listening for messages sent by clients (such as Ping/Pong, Close Frame, or control commands) func readClientMessages(ctx context.Context, conn *websocket.Conn, clientID string, cancel context.CancelFunc) { // conn.SetReadLimit(512) for { @@ -149,54 +133,47 @@ func readClientMessages(ctx context.Context, conn *websocket.Conn, clientID stri } } -// sendAggregateRealTimeDataStream define func to responsible for continuously pushing aggregate real-time data to the client -func sendAggregateRealTimeDataStream(conn *websocket.Conn, targetsData []network.RealTimePullTarget) error { - if len(targetsData) == 0 { - return nil +func flushBuffer(ctx context.Context, buffer *[]network.RealTimePullTarget, sendChan chan<- network.WSResponse, clientID string, reason string) { + if len(*buffer) == 0 { + return } - response := network.SuccessResponse{ - Code: 200, - Msg: "success", + + resp := network.WSResponse{ + Code: constants.RespCodeSuccess, + Msg: "process completed", Payload: network.RealTimePullPayload{ - Targets: targetsData, + Targets: *buffer, }, } - return conn.WriteJSON(response) + + select { + case sendChan <- resp: + default: + logger.Warn(ctx, "sendChan blocked, dropping data batch", "client_id", clientID, "reason", reason) + } + *buffer = make([]network.RealTimePullTarget, 0, constants.SendMaxBatchSize) } // sendDataStream define func to manages a dedicated goroutine to push data batches or system signals to the websocket client -func sendDataStream(ctx context.Context, conn *websocket.Conn, clientID string, sendChan <-chan []network.RealTimePullTarget, cancel context.CancelFunc) { - logger.Info(ctx, "start dedicated websocket sender goroutine", "client_id", clientID) - for targetsData := range sendChan { - // TODO 使用 constants.SysCtrlPrefix + switch-case 形式应对可能的业务扩展 - if len(targetsData) == 1 && targetsData[0].ID == constants.SysCtrlAllRemoved { - err := conn.WriteJSON(map[string]any{ - "code": 2101, - "msg": "all targets removed in given client_id", - "payload": map[string]int{ - "active_targets_count": 0, - }, - }) - if err != nil { - logger.Error(ctx, "send all targets removed system signal failed", "client_id", clientID, "error", err) - cancel() - } - continue +func sendDataStream(ctx context.Context, conn *websocket.Conn, clientID string, sendChan <-chan network.WSResponse, cancel context.CancelFunc) { + defer func() { + if r := recover(); r != nil { + logger.Error(ctx, "sendDataStream recovered from panic", "err", r) } + }() - if err := sendAggregateRealTimeDataStream(conn, targetsData); err != nil { - logger.Error(ctx, "send the real time aggregate data failed in sender goroutine", "client_id", clientID, "error", err) + logger.Info(ctx, "start dedicated websocket sender goroutine", "client_id", clientID) + for resp := range sendChan { + if err := conn.WriteJSON(resp); err != nil { + logger.Error(ctx, "websocket write failed", "client_id", clientID, "error", err) cancel() return } } - logger.Info(ctx, "sender goroutine exiting as channel is closed", "client_id", clientID) } // processTargetPolling define func to process target in subscription map and data is continuously retrieved from redis based on the target -func processTargetPolling(ctx context.Context, s *SharedSubState, clientID string, fanInChan chan network.RealTimePullTarget, sendChan chan<- []network.RealTimePullTarget) { - // ensure the fanInChan will not leak - defer close(fanInChan) +func processTargetPolling(ctx context.Context, s *SharedSubState, clientID string, fanInChan chan network.RealTimePullTarget, sendChan chan<- network.WSResponse) { logger.Info(ctx, fmt.Sprintf("start processing real time data polling for clientID:%s", clientID)) stopChanMap := make(map[string]chan struct{}) s.globalMutex.RLock() @@ -383,7 +360,7 @@ func updateTargets(ctx context.Context, config *RealTimeSubConfig, stopChanMap m } // removeTargets define func to stops running polling goroutines for targets that were removed -func removeTargets(ctx context.Context, stopChanMap map[string]chan struct{}, removeTargets []string, sendChan chan<- []network.RealTimePullTarget) { +func removeTargets(ctx context.Context, stopChanMap map[string]chan struct{}, removeTargets []string, sendChan chan<- network.WSResponse) { for _, target := range removeTargets { stopChan, exists := stopChanMap[target] if !exists { @@ -402,17 +379,18 @@ func removeTargets(ctx context.Context, stopChanMap map[string]chan struct{}, re } } -func sendSpecialStatusToClient(ctx context.Context, sendChan chan<- []network.RealTimePullTarget) { - specialTarget := network.RealTimePullTarget{ - ID: constants.SysCtrlAllRemoved, - Datas: []network.RealTimePullData{}, +func sendSpecialStatusToClient(ctx context.Context, sendChan chan<- network.WSResponse) { + // TODO 使用 constants.SysCtrlPrefix + switch-case 形式应对可能的业务扩展 + resp := network.WSResponse{ + Code: constants.RespCodeSuccessWithNoSub, + Msg: "all targets removed", + Payload: map[string]int{"active_targets_count": 0}, } select { - case sendChan <- []network.RealTimePullTarget{specialTarget}: - logger.Info(ctx, "sent 2101 status request to sendChan") + case sendChan <- resp: default: - logger.Warn(ctx, "sendChan is full, skipping 2101 status message") + logger.Warn(ctx, "sendChan is full, skipping 2101 status") } } @@ -423,7 +401,6 @@ func stopAllPolling(ctx context.Context, stopChanMap map[string]chan struct{}) { close(stopChan) } clear(stopChanMap) - return } // redisPollingConfig define struct for param which query real time data from redis @@ -463,7 +440,7 @@ func realTimeDataQueryFromRedis(ctx context.Context, config redisPollingConfig, } func performQuery(ctx context.Context, client *diagram.RedisClient, config redisPollingConfig, fanInChan chan network.RealTimePullTarget) { - members, err := client.QueryByZRangeByLex(ctx, config.queryKey, config.dataSize) + members, err := client.QueryByZRange(ctx, config.queryKey, config.dataSize) if err != nil { logger.Error(ctx, "query real time data from redis failed", "key", config.queryKey, "error", err) return diff --git a/handler/real_time_data_query.go b/handler/real_time_data_query.go index c734739..03e9786 100644 --- a/handler/real_time_data_query.go +++ b/handler/real_time_data_query.go @@ -168,7 +168,6 @@ func receiveRealTimeDataByWebSocket(ctx context.Context, params url.Values, tran } transportChannel <- subPoss } - return } // messageTypeToString define func of auxiliary to convert message type to string diff --git a/handler/real_time_data_receive.go b/handler/real_time_data_receive.go index 77ab71f..de09a8a 100644 --- a/handler/real_time_data_receive.go +++ b/handler/real_time_data_receive.go @@ -64,7 +64,7 @@ func RealTimeDataReceivehandler(c *gin.Context) { realtimedata.RealTimeDataChan <- request - payload := map[string]interface{}{ + payload := map[string]any{ "component_uuid": request.PayLoad.ComponentUUID, "point": request.PayLoad.Point, } @@ -82,8 +82,8 @@ func RealTimeDataReceivehandler(c *gin.Context) { } } -func processResponse(code int64, msg string, payload map[string]interface{}) []byte { - resp := map[string]interface{}{ +func processResponse(code int64, msg string, payload map[string]any) []byte { + resp := map[string]any{ "code": code, "msg": msg, "payload": payload, diff --git a/handler/real_time_data_subscription.go b/handler/real_time_data_subscription.go index f0f435b..340f48b 100644 --- a/handler/real_time_data_subscription.go +++ b/handler/real_time_data_subscription.go @@ -5,9 +5,9 @@ import ( "context" "fmt" "maps" - "net/http" "sync" + "modelRT/common" "modelRT/constants" "modelRT/database" "modelRT/logger" @@ -33,42 +33,42 @@ func init() { // @Accept json // @Produce json // @Param request body network.RealTimeSubRequest true "量测节点实时数据订阅" -// @Success 200 {object} network.SuccessResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" +// @Success 2000 {object} network.SuccessResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" // -// @Example 200 { -// "code": 200, -// "msg": "success", +// @Example 2000 { +// "code": 2000, +// "msg": "process completed", // "payload": { // "targets": [ // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_C_rms", -// "code": "1001", +// "code": "20000", // "msg": "subscription success" // }, // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_B_rms", -// "code": "1002", +// "code": "20000", // "msg": "subscription failed" // } // ] // } // } // -// @Failure 400 {object} network.FailureResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" +// @Failure 3000 {object} network.FailureResponse{payload=network.RealTimeSubPayload} "订阅实时数据结果列表" // -// @Example 400 { -// "code": 400, -// "msg": "failed to get recommend data from redis", +// @Example 3000 { +// "code": 3000, +// "msg": "process completed with partial failures", // "payload": { // "targets": [ // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_A_rms", -// "code": "1002", +// "code": "40005", // "msg": "subscription failed" // }, // { // "id": "grid1.zone1.station1.ns1.tag1.bay.I11_B_rms", -// "code": "1002", +// "code": "50001", // "msg": "subscription failed" // } // ] @@ -83,10 +83,7 @@ func RealTimeSubHandler(c *gin.Context) { if err := c.ShouldBindJSON(&request); err != nil { logger.Error(c, "failed to unmarshal real time query request", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil) return } @@ -95,10 +92,7 @@ func RealTimeSubHandler(c *gin.Context) { id, err := uuid.NewV4() if err != nil { logger.Error(c, "failed to generate client id", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - }) + renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil) return } clientID = id.String() @@ -123,110 +117,74 @@ func RealTimeSubHandler(c *gin.Context) { results, err := globalSubState.CreateConfig(c, tx, clientID, request.Measurements) if err != nil { logger.Error(c, "create real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "process completed", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return case constants.SubStopAction: results, err := globalSubState.RemoveTargets(c, clientID, request.Measurements) if err != nil { logger.Error(c, "remove target to real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "success", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return case constants.SubAppendAction: results, err := globalSubState.AppendTargets(c, tx, clientID, request.Measurements) if err != nil { logger.Error(c, "append target to real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "success", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return case constants.SubUpdateAction: results, err := globalSubState.UpdateTargets(c, tx, clientID, request.Measurements) if err != nil { logger.Error(c, "update target to real time data subscription config failed", "error", err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespFailure(c, constants.RespCodeFailed, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } - c.JSON(http.StatusOK, network.SuccessResponse{ - Code: http.StatusOK, - Msg: "success", - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + renderRespSuccess(c, constants.RespCodeSuccess, "success", network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return default: - err := fmt.Errorf("%w: request action is %s", constants.ErrUnsupportedSubAction, request.Action) + err := fmt.Errorf("%w: request action is %s", common.ErrUnsupportedSubAction, request.Action) logger.Error(c, "unsupported action of real time data subscription request", "error", err) requestTargetsCount := processRealTimeRequestCount(request.Measurements) - results := processRealTimeRequestTargets(request.Measurements, requestTargetsCount, err) - c.JSON(http.StatusOK, network.FailureResponse{ - Code: http.StatusBadRequest, - Msg: err.Error(), - Payload: network.RealTimeSubPayload{ - ClientID: clientID, - TargetResults: results, - }, + results := processRealTimeRequestTargets(request.Measurements, requestTargetsCount, constants.CodeUnsupportSubOperation, err) + renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), network.RealTimeSubPayload{ + ClientID: clientID, + TargetResults: results, }) return } @@ -283,12 +241,12 @@ func processAndValidateTargetsForStart(ctx context.Context, tx *gorm.DB, measure targetModel, err := database.ParseDataIdentifierToken(ctx, tx, target) if err != nil { logger.Error(ctx, "parse data indentity token failed", "error", err, "identity_token", target) - targetResult.Code = constants.SubFailedCode + targetResult.Code = constants.CodeFoundTargetFailed targetResult.Msg = fmt.Sprintf("%s: %s", constants.SubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) continue } - targetResult.Code = constants.SubSuccessCode + targetResult.Code = constants.CodeSuccess targetResult.Msg = constants.SubSuccessMsg targetProcessResults = append(targetProcessResults, targetResult) successfulTargets = append(successfulTargets, target) @@ -327,7 +285,7 @@ func processAndValidateTargetsForUpdate(ctx context.Context, tx *gorm.DB, config if _, exist := config.targetContext[target]; !exist { err := fmt.Errorf("target %s does not exists in subscription list", target) logger.Error(ctx, "update target does not exist in subscription list", "error", err, "target", target) - targetResult.Code = constants.UpdateSubFailedCode + targetResult.Code = constants.CodeUpdateSubTargetMissing targetResult.Msg = fmt.Sprintf("%s: %s", constants.UpdateSubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) continue @@ -336,13 +294,13 @@ func processAndValidateTargetsForUpdate(ctx context.Context, tx *gorm.DB, config targetModel, err := database.ParseDataIdentifierToken(ctx, tx, target) if err != nil { logger.Error(ctx, "parse data indentity token failed", "error", err, "identity_token", target) - targetResult.Code = constants.UpdateSubFailedCode + targetResult.Code = constants.CodeDBQueryFailed targetResult.Msg = fmt.Sprintf("%s: %s", constants.UpdateSubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) continue } - targetResult.Code = constants.UpdateSubSuccessCode + targetResult.Code = constants.CodeSuccess targetResult.Msg = constants.UpdateSubSuccessMsg targetProcessResults = append(targetProcessResults, targetResult) successfulTargets = append(successfulTargets, target) @@ -473,7 +431,7 @@ func (s *SharedSubState) AppendTargets(ctx context.Context, tx *gorm.DB, clientI if !exist { err := fmt.Errorf("clientID %s not found. use CreateConfig to start a new config", clientID) logger.Error(ctx, "clientID not found. use CreateConfig to start a new config", "error", err) - return processRealTimeRequestTargets(measurements, requestTargetsCount, err), err + return processRealTimeRequestTargets(measurements, requestTargetsCount, constants.CodeAppendSubTargetMissing, err), err } targetProcessResults, successfulTargets, newMeasMap, newMeasContextMap := processAndValidateTargetsForStart(ctx, tx, measurements, requestTargetsCount) @@ -507,7 +465,7 @@ func filterAndDeduplicateRepeatTargets(resultsSlice []network.TargetResult, idsS for index := range resultsSlice { if _, isTarget := set[resultsSlice[index].ID]; isTarget { - resultsSlice[index].Code = constants.SubRepeatCode + resultsSlice[index].Code = constants.CodeSubTargetRepeat resultsSlice[index].Msg = constants.SubRepeatMsg } } @@ -575,7 +533,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea s.globalMutex.RUnlock() err := fmt.Errorf("clientID %s not found", clientID) logger.Error(ctx, "clientID not found in remove targets operation", "error", err) - return processRealTimeRequestTargets(measurements, requestTargetsCount, err), err + return processRealTimeRequestTargets(measurements, requestTargetsCount, constants.CodeCancelSubTargetMissing, err), err } s.globalMutex.RUnlock() @@ -595,7 +553,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea for _, target := range measTargets { targetResult := network.TargetResult{ ID: target, - Code: constants.CancelSubFailedCode, + Code: constants.CodeCancelSubTargetMissing, Msg: constants.CancelSubFailedMsg, } targetProcessResults = append(targetProcessResults, targetResult) @@ -616,7 +574,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea transportTargets.Targets = append(transportTargets.Targets, existingTarget) targetResult := network.TargetResult{ ID: existingTarget, - Code: constants.CancelSubSuccessCode, + Code: constants.CodeSuccess, Msg: constants.CancelSubSuccessMsg, } targetProcessResults = append(targetProcessResults, targetResult) @@ -639,7 +597,7 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea for target := range targetsToRemoveMap { targetResult := network.TargetResult{ ID: target, - Code: constants.CancelSubFailedCode, + Code: constants.CodeCancelSubTargetMissing, Msg: fmt.Sprintf("%s: %s", constants.SubFailedMsg, err.Error()), } targetProcessResults = append(targetProcessResults, targetResult) @@ -663,17 +621,15 @@ func (s *SharedSubState) RemoveTargets(ctx context.Context, clientID string, mea // UpdateTargets define function to update targets in SharedSubState func (s *SharedSubState) UpdateTargets(ctx context.Context, tx *gorm.DB, clientID string, measurements []network.RealTimeMeasurementItem) ([]network.TargetResult, error) { requestTargetsCount := processRealTimeRequestCount(measurements) - targetProcessResults := make([]network.TargetResult, 0, requestTargetsCount) s.globalMutex.RLock() config, exist := s.subMap[clientID] s.globalMutex.RUnlock() if !exist { - s.globalMutex.RUnlock() err := fmt.Errorf("clientID %s not found", clientID) logger.Error(ctx, "clientID not found in remove targets operation", "error", err) - return processRealTimeRequestTargets(measurements, requestTargetsCount, err), err + return processRealTimeRequestTargets(measurements, requestTargetsCount, constants.CodeUpdateSubTargetMissing, err), err } targetProcessResults, successfulTargets, newMeasMap, newMeasContextMap := processAndValidateTargetsForUpdate(ctx, tx, config, measurements, requestTargetsCount) @@ -722,13 +678,13 @@ func processRealTimeRequestCount(measurements []network.RealTimeMeasurementItem) return totalTargetsCount } -func processRealTimeRequestTargets(measurements []network.RealTimeMeasurementItem, targetCount int, err error) []network.TargetResult { +func processRealTimeRequestTargets(measurements []network.RealTimeMeasurementItem, targetCount int, businessCode int, err error) []network.TargetResult { targetProcessResults := make([]network.TargetResult, 0, targetCount) for _, measurementItem := range measurements { for _, target := range measurementItem.Targets { var targetResult network.TargetResult targetResult.ID = target - targetResult.Code = constants.SubFailedCode + targetResult.Code = businessCode targetResult.Msg = fmt.Sprintf("%s: %s", constants.SubFailedMsg, err.Error()) targetProcessResults = append(targetProcessResults, targetResult) } diff --git a/logger/facede.go b/logger/facede.go index b9a5b54..1533cdf 100644 --- a/logger/facede.go +++ b/logger/facede.go @@ -39,11 +39,30 @@ func Error(ctx context.Context, msg string, kv ...any) { } func (f *facade) log(ctx context.Context, lvl zapcore.Level, msg string, kv ...any) { - fields := makeLogFields(ctx, kv...) + f.logSkip(ctx, lvl, 0, msg, kv...) +} + +func (f *facade) logSkip(ctx context.Context, lvl zapcore.Level, extraSkip int, msg string, kv ...any) { + fields := makeLogFieldsSkip(ctx, extraSkip, kv...) ce := f._logger.Check(lvl, msg) ce.Write(fields...) } +// ErrorSkip logs at error level with extra caller skip frames for wrapper functions. +func ErrorSkip(ctx context.Context, extraSkip int, msg string, kv ...any) { + logFacade().logSkip(ctx, zapcore.ErrorLevel, extraSkip, msg, kv...) +} + +// WarnSkip logs at warn level with extra caller skip frames for wrapper functions. +func WarnSkip(ctx context.Context, extraSkip int, msg string, kv ...any) { + logFacade().logSkip(ctx, zapcore.WarnLevel, extraSkip, msg, kv...) +} + +// InfoSkip logs at info level with extra caller skip frames for wrapper functions. +func InfoSkip(ctx context.Context, extraSkip int, msg string, kv ...any) { + logFacade().logSkip(ctx, zapcore.InfoLevel, extraSkip, msg, kv...) +} + func logFacade() *facade { fOnce.Do(func() { f = &facade{ diff --git a/logger/gorm_logger.go b/logger/gorm_logger.go index c5e6c01..fadd2e6 100644 --- a/logger/gorm_logger.go +++ b/logger/gorm_logger.go @@ -48,14 +48,13 @@ func (l *GormLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql duration := time.Since(begin).Milliseconds() // get gorm exec sql and rows affected sql, rows := fc() - // gorm error judgment if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - Error(ctx, "SQL ERROR", "sql", sql, "rows", rows, "dur(ms)", duration) + ErrorSkip(ctx, 1, "SQL ERROR", "sql", sql, "rows", rows, "dur(ms)", duration) + return } - // slow query judgment if duration > l.SlowThreshold.Milliseconds() { - Warn(ctx, "SQL SLOW", "sql", sql, "rows", rows, "dur(ms)", duration) + WarnSkip(ctx, 1, "SQL SLOW", "sql", sql, "rows", rows, "dur(ms)", duration) } else { - Info(ctx, "SQL INFO", "sql", sql, "rows", rows, "dur(ms)", duration) + InfoSkip(ctx, 1, "SQL INFO", "sql", sql, "rows", rows, "dur(ms)", duration) } } diff --git a/logger/logger.go b/logger/logger.go index 116e3b1..f7fa247 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -6,17 +6,21 @@ import ( "path" "runtime" - "modelRT/constants" - + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) +// Logger is the interface returned by New for structured, trace-aware logging. +type Logger interface { + Debug(msg string, kv ...any) + Info(msg string, kv ...any) + Warn(msg string, kv ...any) + Error(msg string, kv ...any) +} + type logger struct { ctx context.Context - traceID string - spanID string - pSpanID string _logger *zap.Logger } @@ -43,14 +47,20 @@ func (l *logger) log(lvl zapcore.Level, msg string, kv ...any) { } func makeLogFields(ctx context.Context, kv ...any) []zap.Field { - // Ensure that log information appears in pairs in the form of key-value pairs + return makeLogFieldsSkip(ctx, 0, kv...) +} + +func makeLogFieldsSkip(ctx context.Context, extraSkip int, kv ...any) []zap.Field { if len(kv)%2 != 0 { kv = append(kv, "unknown") } - kv = append(kv, "traceID", ctx.Value(constants.HeaderTraceID), "spanID", ctx.Value(constants.HeaderSpanID), "parentSpanID", ctx.Value(constants.HeaderParentSpanID)) + spanCtx := trace.SpanFromContext(ctx).SpanContext() + traceID := spanCtx.TraceID().String() + spanID := spanCtx.SpanID().String() + kv = append(kv, "traceID", traceID, "spanID", spanID) - funcName, file, line := getLoggerCallerInfo() + funcName, file, line := getLoggerCallerInfoSkip(extraSkip) kv = append(kv, "func", funcName, "file", file, "line", line) fields := make([]zap.Field, 0, len(kv)/2) for i := 0; i < len(kv); i += 2 { @@ -78,9 +88,14 @@ func makeLogFields(ctx context.Context, kv ...any) []zap.Field { return fields } -// getLoggerCallerInfo define func of return log caller information、method name、file name、line number +// getLoggerCallerInfo returns caller info at a fixed depth for the standard facade call chain. func getLoggerCallerInfo() (funcName, file string, line int) { - pc, file, line, ok := runtime.Caller(4) + return getLoggerCallerInfoSkip(0) +} + +// getLoggerCallerInfoSkip returns caller info with additional skip frames beyond the standard depth. +func getLoggerCallerInfoSkip(extraSkip int) (funcName, file string, line int) { + pc, file, line, ok := runtime.Caller(4 + extraSkip) if !ok { return } @@ -89,23 +104,11 @@ func getLoggerCallerInfo() (funcName, file string, line int) { return } -func New(ctx context.Context) *logger { - var traceID, spanID, pSpanID string - if ctx.Value("traceID") != nil { - traceID = ctx.Value("traceID").(string) - } - if ctx.Value("spanID") != nil { - spanID = ctx.Value("spanID").(string) - } - if ctx.Value("psapnID") != nil { - pSpanID = ctx.Value("pspanID").(string) - } - +// New returns a logger bound to ctx. Trace fields (traceID, spanID) are extracted +// from the OTel span stored in ctx and included in every log entry. +func New(ctx context.Context) Logger { return &logger{ ctx: ctx, - traceID: traceID, - spanID: spanID, - pSpanID: pSpanID, _logger: GetLoggerInstance(), } } diff --git a/logger/loki_syncer.go b/logger/loki_syncer.go new file mode 100644 index 0000000..332ddcf --- /dev/null +++ b/logger/loki_syncer.go @@ -0,0 +1,132 @@ +// Package logger define log struct of modelRT project +package logger + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "maps" + "net/http" + "os" + "strconv" + "sync" + "time" + + "modelRT/config" +) + +type lokiPushRequest struct { + Streams []lokiStream `json:"streams"` +} + +type lokiStream struct { + Stream map[string]string `json:"stream"` + Values [][2]string `json:"values"` +} + +// lokiSyncer implements zapcore.WriteSyncer, batching log lines and pushing them +// to Loki's push API asynchronously. Errors are silently dropped so a unreachable +// Loki instance never blocks or crashes the application. +type lokiSyncer struct { + endpoint string + labels map[string]string + client *http.Client + ch chan string + wg sync.WaitGroup + closeOnce sync.Once +} + +func newLokiSyncer(lCfg config.LokiConfig) *lokiSyncer { + // always tag development logs with env=development; caller-supplied labels override if needed + labels := map[string]string{"env": "development"} + maps.Copy(labels, lCfg.Labels) + ls := &lokiSyncer{ + endpoint: lCfg.Endpoint + "/loki/api/v1/push", + labels: labels, + client: &http.Client{Timeout: 5 * time.Second}, + ch: make(chan string, 512), + } + ls.wg.Add(1) + go ls.run() + return ls +} + +func (ls *lokiSyncer) Write(p []byte) (int, error) { + select { + case ls.ch <- string(p): + default: + // channel full: drop the line rather than block the caller + } + return len(p), nil +} + +// Sync flushes remaining buffered lines and shuts down the background goroutine. +// Called by zap.Logger.Sync() at application shutdown. +func (ls *lokiSyncer) Sync() error { + ls.closeOnce.Do(func() { close(ls.ch) }) + ls.wg.Wait() + return nil +} + +func (ls *lokiSyncer) run() { + defer ls.wg.Done() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + var batch []string + flush := func() { + if len(batch) == 0 { + return + } + ls.push(batch) + batch = batch[:0] + } + + for { + select { + case line, ok := <-ls.ch: + if !ok { + flush() + return + } + batch = append(batch, line) + if len(batch) >= 100 { + flush() + } + case <-ticker.C: + flush() + } + } +} + +func (ls *lokiSyncer) push(lines []string) { + ts := strconv.FormatInt(time.Now().UnixNano(), 10) + values := make([][2]string, len(lines)) + for i, line := range lines { + values[i] = [2]string{ts, line} + } + + body, err := json.Marshal(lokiPushRequest{ + Streams: []lokiStream{{Stream: ls.labels, Values: values}}, + }) + if err != nil { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, ls.endpoint, bytes.NewReader(body)) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + + resp, err := ls.client.Do(req) + if err != nil { + fmt.Fprintf(os.Stderr, "loki syncer: push failed: %v\n", err) + return + } + defer resp.Body.Close() +} diff --git a/logger/zap.go b/logger/zap.go index 5e65db7..52d74eb 100644 --- a/logger/zap.go +++ b/logger/zap.go @@ -21,53 +21,89 @@ var ( _globalLogger *zap.Logger ) -// getEncoder responsible for setting the log format for encoding -func getEncoder() zapcore.Encoder { - encoderConfig := zap.NewProductionEncoderConfig() - // serialization time eg:2006-01-02 15:04:05 - encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05") - encoderConfig.TimeKey = "time" - encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder - encoderConfig.EncodeCaller = zapcore.ShortCallerEncoder - return zapcore.NewJSONEncoder(encoderConfig) +// getEncoder returns a console encoder for development (human-readable, colored) and a JSON encoder +// for container modes (parseable by Promtail pipeline_stages). +func getEncoder(mode string) zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05") + cfg.TimeKey = "time" + cfg.EncodeCaller = zapcore.ShortCallerEncoder + if mode == constants.DevelopmentLogMode { + cfg.EncodeLevel = zapcore.CapitalColorLevelEncoder + return zapcore.NewConsoleEncoder(cfg) + } + cfg.EncodeLevel = zapcore.CapitalLevelEncoder + return zapcore.NewJSONEncoder(cfg) } -// getLogWriter responsible for setting the location of log storage -func getLogWriter(mode, filename string, maxsize, maxBackup, maxAge int, compress bool) zapcore.WriteSyncer { - dateStr := time.Now().Format("2006-01-02 15:04:05") - finalFilename := fmt.Sprintf(filename, dateStr) - lumberJackLogger := &lumberjack.Logger{ - Filename: finalFilename, // log file position - MaxSize: maxsize, // log file maxsize - MaxAge: maxAge, // maximum number of day files retained - MaxBackups: maxBackup, // maximum number of old files retained - Compress: compress, // whether to compress +// getWriteSyncer returns write targets based on mode: +// - development: stdout + optional Loki direct-push (when loki.endpoint is set) +// - container modes: stdout always (Promtail collects) + rotating file (when filepath is set) +func getWriteSyncer(lCfg config.LoggerConfig) zapcore.WriteSyncer { + stdout := zapcore.AddSync(os.Stdout) + + if lCfg.Mode == constants.DevelopmentLogMode { + if lCfg.Loki.Endpoint == "" { + return stdout + } + return zapcore.NewMultiWriteSyncer(stdout, newLokiSyncer(lCfg.Loki)) } - syncConsole := zapcore.AddSync(os.Stderr) - if mode == constants.DevelopmentLogMode { - return syncConsole + syncers := []zapcore.WriteSyncer{stdout} + if lCfg.FilePath != "" { + dateStr := time.Now().Format("2006-01-02 15:04:05") + syncers = append(syncers, zapcore.AddSync(&lumberjack.Logger{ + Filename: fmt.Sprintf(lCfg.FilePath, dateStr), + MaxSize: lCfg.MaxSize, + MaxAge: lCfg.MaxAge, + MaxBackups: lCfg.MaxBackups, + Compress: lCfg.Compress, + })) } + return zapcore.NewMultiWriteSyncer(syncers...) +} - syncFile := zapcore.AddSync(lumberJackLogger) - return zapcore.NewMultiWriteSyncer(syncFile, syncConsole) +// containerFields reads K8s Downward API environment variables and returns them as global zap fields. +// These fields appear on every log line, allowing Loki/Grafana to filter by pod, namespace, and node. +// Inject them in the Deployment manifest: +// +// env: +// - name: K8S_NAMESPACE +// valueFrom: {fieldRef: {fieldPath: metadata.namespace}} +// - name: K8S_NODE_NAME +// valueFrom: {fieldRef: {fieldPath: spec.nodeName}} +func containerFields() []zap.Field { + var fields []zap.Field + // HOSTNAME is automatically set to the pod name by Kubernetes. + if pod := os.Getenv("HOSTNAME"); pod != "" { + fields = append(fields, zap.String("pod", pod)) + } + if ns := os.Getenv("K8S_NAMESPACE"); ns != "" { + fields = append(fields, zap.String("namespace", ns)) + } + if node := os.Getenv("K8S_NODE_NAME"); node != "" { + fields = append(fields, zap.String("node", node)) + } + return fields } // initLogger return successfully initialized zap logger func initLogger(lCfg config.LoggerConfig) *zap.Logger { - writeSyncer := getLogWriter(lCfg.Mode, lCfg.FilePath, lCfg.MaxSize, lCfg.MaxBackups, lCfg.MaxAge, lCfg.Compress) - encoder := getEncoder() + writeSyncer := getWriteSyncer(lCfg) + encoder := getEncoder(lCfg.Mode) l := new(zapcore.Level) - err := l.UnmarshalText([]byte(lCfg.Level)) - if err != nil { + if err := l.UnmarshalText([]byte(lCfg.Level)); err != nil { panic(err) } core := zapcore.NewCore(encoder, writeSyncer, l) - logger := zap.New(core, zap.AddCaller()) + opts := []zap.Option{zap.AddCaller()} + if lCfg.Mode != constants.DevelopmentLogMode { + opts = append(opts, zap.Fields(containerFields()...)) + } + logger := zap.New(core, opts...) - // 替换全局日志实例 zap.ReplaceGlobals(logger) return logger } diff --git a/main.go b/main.go index a875a74..259b269 100644 --- a/main.go +++ b/main.go @@ -12,17 +12,24 @@ import ( "os/signal" "path/filepath" "syscall" + "time" - "modelRT/alert" "modelRT/config" + "modelRT/constants" "modelRT/database" "modelRT/diagram" "modelRT/logger" + "modelRT/middleware" "modelRT/model" + "modelRT/mq" "modelRT/pool" + "modelRT/real-time-data/alert" "modelRT/router" + "modelRT/task" "modelRT/util" + "github.com/gin-contrib/cors" + locker "modelRT/distributedlock" _ "modelRT/docs" @@ -32,6 +39,7 @@ import ( "github.com/panjf2000/ants/v2" swaggerFiles "github.com/swaggo/files" ginSwagger "github.com/swaggo/gin-swagger" + "go.opentelemetry.io/otel" "gorm.io/gorm" ) @@ -64,9 +72,9 @@ var ( // // @host localhost:8080 // @BasePath /api/v1 + func main() { flag.Parse() - ctx := context.TODO() configPath := filepath.Join(*modelRTConfigDir, *modelRTConfigName+"."+*modelRTConfigType) if _, err := os.Stat(configPath); os.IsNotExist(err) { @@ -92,13 +100,29 @@ func main() { logger.InitLoggerInstance(modelRTConfig.LoggerConfig) defer logger.GetLoggerInstance().Sync() + // init OTel TracerProvider + tp, tpErr := middleware.InitTracerProvider(context.Background(), modelRTConfig) + if tpErr != nil { + log.Printf("warn: OTLP tracer init failed, tracing disabled: %v", tpErr) + } + if tp != nil { + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + tp.Shutdown(shutdownCtx) + }() + } + + ctx, startupSpan := otel.Tracer("modelRT/main").Start(context.Background(), "startup") + defer startupSpan.End() + hostName, err := os.Hostname() if err != nil { logger.Error(ctx, "get host name failed", "error", err) panic(err) } - serviceToken, err := util.GenerateClientToken(hostName, modelRTConfig.ServiceConfig.ServiceName, modelRTConfig.ServiceConfig.SecretKey) + serviceToken, err := util.GenerateClientToken(hostName, modelRTConfig.ServiceName, modelRTConfig.SecretKey) if err != nil { logger.Error(ctx, "generate client token failed", "error", err) panic(err) @@ -127,13 +151,17 @@ func main() { defer parsePool.Release() searchPool, err := util.NewRedigoPool(modelRTConfig.StorageRedisConfig) + if err != nil { + logger.Error(ctx, "init redigo pool failed", "error", err) + panic(err) + } defer searchPool.Close() model.InitAutocompleterWithPool(searchPool) - storageClient := diagram.InitRedisClientInstance(modelRTConfig.StorageRedisConfig) + storageClient := diagram.InitRedisClientInstance(modelRTConfig.StorageRedisConfig, modelRTConfig.DeployEnv) defer storageClient.Close() - lockerClient := locker.InitClientInstance(modelRTConfig.LockerRedisConfig) + lockerClient := locker.InitClientInstance(modelRTConfig.LockerRedisConfig, modelRTConfig.DeployEnv) defer lockerClient.Close() // init anchor param ants pool @@ -144,7 +172,28 @@ func main() { } defer anchorRealTimePool.Release() - postgresDBClient.Transaction(func(tx *gorm.DB) error { + // init rabbitmq connection + mq.InitRabbitProxy(ctx, modelRTConfig.RabbitMQConfig) + defer mq.CloseRabbitProxy() + + // init async task worker + taskWorker, err := task.InitTaskWorker(ctx, modelRTConfig, postgresDBClient) + if err != nil { + logger.Error(ctx, "Failed to initialize task worker", "error", err) + // Continue without task worker, but log warning + } else { + go taskWorker.Start() + defer taskWorker.Stop() + } + + // async push event to rabbitMQ + go mq.PushUpDownLimitEventToRabbitMQ(ctx, mq.MsgChan) + // async push message (task state changes etc.) to rabbitMQ + go mq.PushMessageToRabbitMQ(ctx, mq.MessageMsgChan) + // async push task message to rabbitMQ + go task.PushTaskToRabbitMQ(ctx, modelRTConfig.RabbitMQConfig, task.TaskMsgChan) + + postgresDBClient.WithContext(ctx).Transaction(func(tx *gorm.DB) error { // load circuit diagram from postgres // componentTypeMap, err := database.QueryCircuitDiagramComponentFromDB(cancelCtx, tx, parsePool) // if err != nil { @@ -165,7 +214,7 @@ func main() { panic(err) } - measurementSet, err := database.GetFullMeasurementSet(tx) + measurementSet, err := database.GetFullMeasurementSet(ctx, postgresDBClient) if err != nil { logger.Error(ctx, "generate component measurement group failed", "error", err) panic(err) @@ -193,9 +242,9 @@ func main() { logger.Error(ctx, "load topologic info from postgres failed", "error", err) panic(err) } - go realtimedata.StartRealTimeDataComputing(ctx, allMeasurement) + go realtimedata.StartComputingRealTimeDataLimit(ctx, allMeasurement) - tree, err := database.QueryTopologicFromDB(ctx, tx) + tree, _, err := database.QueryTopologicFromDB(ctx, tx) if err != nil { logger.Error(ctx, "load topologic info from postgres failed", "error", err) panic(err) @@ -204,26 +253,28 @@ func main() { return nil }) - // use release mode in productio - // gin.SetMode(gin.ReleaseMode) + // use release mode in production + if modelRTConfig.DeployEnv == constants.ProductionDeployMode { + gin.SetMode(gin.ReleaseMode) + } engine := gin.New() + // 添加CORS中间件 + engine.Use(cors.New(cors.Config{ + AllowOrigins: []string{"*"}, // 或指定具体域名 + AllowMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + AllowHeaders: []string{"Origin", "Content-Type", "Authorization"}, + ExposeHeaders: []string{"Content-Length"}, + AllowCredentials: true, + MaxAge: 12 * time.Hour, + })) router.RegisterRoutes(engine, serviceToken) - // Swagger UI - engine.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler)) - - // 注册 Swagger UI 路由 - // docs.SwaggerInfo.BasePath = "/model" - // v1 := engine.Group("/api/v1") - // { - // eg := v1.Group("/example") - // { - // eg.GET("/helloworld", Helloworld) - // } - // } + if modelRTConfig.DeployEnv != constants.ProductionDeployMode { + engine.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler)) + } server := http.Server{ - Addr: modelRTConfig.ServiceConfig.ServiceAddr, + Addr: modelRTConfig.ServiceAddr, Handler: engine, } @@ -232,9 +283,11 @@ func main() { signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) go func() { <-done + logger.Info(ctx, "shutdown signal received, cleaning up...") if err := server.Shutdown(context.Background()); err != nil { logger.Error(ctx, "shutdown serverError", "err", err) } + logger.Info(ctx, "resources cleaned up, exiting") }() logger.Info(ctx, "starting ModelRT server") diff --git a/middleware/config_middleware.go b/middleware/config_middleware.go new file mode 100644 index 0000000..9ff5436 --- /dev/null +++ b/middleware/config_middleware.go @@ -0,0 +1,16 @@ +// Package middleware define gin framework middlewares +package middleware + +import ( + "modelRT/config" + + "github.com/gin-gonic/gin" +) + +// ConfigMiddleware 将全局配置注入到Gin上下文中 +func ConfigMiddleware(modelRTConfig config.ModelRTConfig) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("config", modelRTConfig) + c.Next() + } +} diff --git a/middleware/limiter.go b/middleware/limiter.go index ca8dc53..d1debe1 100644 --- a/middleware/limiter.go +++ b/middleware/limiter.go @@ -1,3 +1,4 @@ +// Package middleware define gin framework middlewares package middleware import ( diff --git a/middleware/panic_recover.go b/middleware/panic_recover.go index e7aa3ac..b8c101f 100644 --- a/middleware/panic_recover.go +++ b/middleware/panic_recover.go @@ -1,3 +1,4 @@ +// Package middleware define gin framework middlewares package middleware import ( diff --git a/middleware/token.go b/middleware/token.go index 06ffa79..6759f40 100644 --- a/middleware/token.go +++ b/middleware/token.go @@ -1,3 +1,4 @@ +// Package middleware define gin framework middlewares package middleware import "github.com/gin-gonic/gin" diff --git a/middleware/trace.go b/middleware/trace.go index 1bb27ca..c113d20 100644 --- a/middleware/trace.go +++ b/middleware/trace.go @@ -1,32 +1,90 @@ +// Package middleware defines gin framework middlewares and OTel tracing infrastructure. package middleware import ( "bytes" + "context" + "fmt" "io" "strings" "time" + "modelRT/config" "modelRT/constants" "modelRT/logger" - "modelRT/util" "github.com/gin-gonic/gin" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + sdkresource "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" ) -// StartTrace define func of set trace info from request header +// InitTracerProvider creates an OTLP TracerProvider and registers it as the global provider. +// It registers the W3C TraceContext propagator (traceparent header). +// The caller is responsible for calling Shutdown on the returned provider during graceful shutdown. +func InitTracerProvider(ctx context.Context, cfg config.ModelRTConfig) (*sdktrace.TracerProvider, error) { + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(cfg.OtelConfig.Endpoint), + } + if cfg.OtelConfig.Insecure { + opts = append(opts, otlptracehttp.WithInsecure()) + } + + exporter, err := otlptracehttp.New(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("create OTLP exporter: %w", err) + } + + res := sdkresource.NewSchemaless( + attribute.String("service.name", cfg.ServiceName), + attribute.String("deployment.environment", cfg.DeployEnv), + ) + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + return tp, nil +} + +// StartTrace extracts upstream W3C trace context from request headers and starts a server span. func StartTrace() gin.HandlerFunc { + tracer := otel.Tracer("modelRT/http") return func(c *gin.Context) { - traceID := c.Request.Header.Get(constants.HeaderTraceID) - parentSpanID := c.Request.Header.Get(constants.HeaderSpanID) - spanID := util.GenerateSpanID(c.Request.RemoteAddr) - // if traceId is empty, it means it is the origin of the link. Set it to the spanId of this time. The originating spanId is the root spanId. - if traceID == "" { - // traceId identifies the entire request link, and spanId identifies the different services in the link. - traceID = spanID + ctx := otel.GetTextMapPropagator().Extract( + c.Request.Context(), + propagation.HeaderCarrier(c.Request.Header), + ) + + spanName := c.FullPath() + if spanName == "" { + spanName = c.Request.URL.Path } - c.Set(constants.HeaderTraceID, traceID) - c.Set(constants.HeaderSpanID, spanID) - c.Set(constants.HeaderParentSpanID, parentSpanID) + ctx, span := tracer.Start(ctx, spanName, + oteltrace.WithSpanKind(oteltrace.SpanKindServer), + ) + defer span.End() + + // backward compat: inject typed keys so existing logger reads work until Step 6 + spanCtx := span.SpanContext() + ctx = context.WithValue(ctx, constants.CtxKeyTraceID, spanCtx.TraceID().String()) + ctx = context.WithValue(ctx, constants.CtxKeySpanID, spanCtx.SpanID().String()) + + c.Request = c.Request.WithContext(ctx) + + // set in gin context for accessLog (logger.New(c) reads via gin.Context.Value) + c.Set(constants.HeaderTraceID, spanCtx.TraceID().String()) + c.Set(constants.HeaderSpanID, spanCtx.SpanID().String()) + c.Next() } } @@ -78,11 +136,10 @@ func LogAccess() gin.HandlerFunc { accessLog(c, "access_end", time.Since(start), reqBody, responseLogging) }() c.Next() - return } } -func accessLog(c *gin.Context, accessType string, dur time.Duration, body []byte, dataOut interface{}) { +func accessLog(c *gin.Context, accessType string, dur time.Duration, body []byte, dataOut any) { req := c.Request bodyStr := string(body) query := req.URL.RawQuery diff --git a/model/attribute_model.go b/model/attribute_model.go index ff7d55e..601d156 100644 --- a/model/attribute_model.go +++ b/model/attribute_model.go @@ -11,7 +11,7 @@ type AttrModelInterface interface { GetZoneInfo() *orm.Zone GetStationInfo() *orm.Station GetComponentInfo() *orm.Component - GetAttrValue() interface{} // New method to get the attribute value + GetAttrValue() any // New method to get the attribute value IsLocal() bool } diff --git a/model/measurement_protol_model.go b/model/measurement_protol_model.go index 9a5446b..7fdce12 100644 --- a/model/measurement_protol_model.go +++ b/model/measurement_protol_model.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "modelRT/common" "modelRT/constants" ) @@ -61,7 +62,7 @@ func generateChannelName(prefix string, number int, suffix string) (string, erro switch prefix { case constants.ChannelPrefixTelemetry: if number > 10 { - return "", constants.ErrExceedsLimitType + return "", common.ErrExceedsLimitType } var builder strings.Builder numberStr := strconv.Itoa(number) @@ -86,7 +87,7 @@ func generateChannelName(prefix string, number int, suffix string) (string, erro channelName := builder.String() return channelName, nil default: - return "", constants.ErrUnsupportedChannelPrefixType + return "", common.ErrUnsupportedChannelPrefixType } } @@ -164,14 +165,14 @@ func (m MeasurementDataSource) GetIOAddress() (IOAddress, error) { if addr, ok := m.IOAddress.(CL3611Address); ok { return addr, nil } - return nil, constants.ErrInvalidAddressType + return nil, common.ErrInvalidAddressType case constants.DataSourceTypePower104: if addr, ok := m.IOAddress.(Power104Address); ok { return addr, nil } - return nil, constants.ErrInvalidAddressType + return nil, common.ErrInvalidAddressType default: - return nil, constants.ErrUnknownDataType + return nil, common.ErrUnknownDataType } } diff --git a/model/recommend_islocal_cache.go b/model/recommend_islocal_cache.go index 8bb7cda..a9f54b5 100644 --- a/model/recommend_islocal_cache.go +++ b/model/recommend_islocal_cache.go @@ -22,7 +22,7 @@ func GetNSpathToIsLocalMap(ctx context.Context, db *gorm.DB) (map[string]bool, e var results []ComponentStationRelation nspathMap := make(map[string]bool) - err := db.Table("component"). + err := db.WithContext(ctx).Table("component"). Select("component.nspath, station.is_local"). Joins("join station on component.station_id = station.id"). Scan(&results).Error diff --git a/model/redis_recommend.go b/model/redis_recommend.go index e0ac507..ddfef2a 100644 --- a/model/redis_recommend.go +++ b/model/redis_recommend.go @@ -550,7 +550,6 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co IsFuzzy: true, Err: nil, } - return } // runFuzzySearch define func to process redis fuzzy search diff --git a/mq/emit.go b/mq/emit.go new file mode 100644 index 0000000..4c3dca6 --- /dev/null +++ b/mq/emit.go @@ -0,0 +1,28 @@ +// Package mq provides read or write access to message queue services +package mq + +import ( + "context" + + "modelRT/logger" + "modelRT/mq/event" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +// TryEmitMessage pushes a message record into MessageMsgChan non-blocking. +// If the channel is full the message is dropped and a warning is logged. +func TryEmitMessage(ctx context.Context, record *event.EventRecord) { + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + msg := &EventMessage{Record: record, TraceCarrier: carrier} + select { + case MessageMsgChan <- msg: + default: + logger.Warn(ctx, "message channel full, message dropped", + "event_uuid", record.EventUUID, + "category", record.Category, + ) + } +} diff --git a/mq/event/event.go b/mq/event/event.go new file mode 100644 index 0000000..951242e --- /dev/null +++ b/mq/event/event.go @@ -0,0 +1,41 @@ +// Package event define real time data evnet operation functions +package event + +// EventRecord define struct for CIM event record +type EventRecord struct { + // 事件名称 + EventName string `json:"event"` + // 事件唯一标识符 + EventUUID string `json:"event_uuid"` + // 事件类型 + Type int `json:"type"` + // 事件优先级 (0-9) + Priority int `json:"priority"` + // 事件状态 + Status int `json:"status"` + // 是否已持久化到数据库，由 eventRT 消费并落库后置为 true + IsPersisted bool `json:"is_persisted"` + // 可选模板参数 + Category string `json:"category,omitempty"` + // 毫秒级时间戳 (Unix epoch) + Timestamp int64 `json:"timestamp"` + // 事件来源 (station, platform, msa) + From string `json:"from"` + // 事件场景描述对象 (如阈值、当前值) + Condition map[string]any `json:"condition"` + // 与事件相关的订阅信息 + AttachedSubscriptions []any `json:"attached_subscriptions"` + // 事件分析结果对象 + Result map[string]any `json:"result,omitempty"` + // 操作历史记录 (CIM ActivityRecord) + Operations []OperationRecord `json:"operations"` + // 子站告警原始数据 (CIM Alarm 数据) + Origin map[string]any `json:"origin,omitempty"` +} + +// OperationRecord 描述对事件的操作记录，如确认(acknowledgment)等 +type OperationRecord struct { + Action string `json:"action"` // 执行的动作，如 "acknowledgment" + Op string `json:"op"` // 操作人/操作账号标识 + TS int64 `json:"ts"` // 操作发生的毫秒时间戳 +} diff --git a/mq/event/event_handlers.go b/mq/event/event_handlers.go new file mode 100644 index 0000000..e37e195 --- /dev/null +++ b/mq/event/event_handlers.go @@ -0,0 +1,82 @@ +// Package event define real time data evnet operation functions +package event + +import ( + "context" + + "modelRT/common" + "modelRT/logger" +) + +type actionHandler func(ctx context.Context, content string, ops ...EventOption) (*EventRecord, error) + +// actionDispatchMap define variable to store all action handler into map +var actionDispatchMap = map[string]actionHandler{ + "info": handleInfoAction, + "warning": handleWarningAction, + "error": handleErrorAction, + "critical": handleCriticalAction, + "exception": handleExceptionAction, +} + +// TriggerEventAction define func to trigger event by action in compute config +func TriggerEventAction(ctx context.Context, command string, eventName string, ops ...EventOption) (*EventRecord, error) { + handler, exists := actionDispatchMap[command] + if !exists { + logger.Error(ctx, "unknown action command", "command", command) + return nil, common.ErrUnknowEventActionCommand + } + + eventRecord, err := handler(ctx, eventName, ops...) + if err != nil { + logger.Error(ctx, "action event handler failed", "error", err) + return nil, common.ErrExecEventActionFailed + } + return eventRecord, nil +} + +func handleInfoAction(ctx context.Context, eventName string, ops ...EventOption) (*EventRecord, error) { + logger.Info(ctx, "trigger info event", "event_name", eventName) + eventRecord, err := NewGeneralPlatformSoftRecord(eventName, ops...) + if err != nil { + logger.Error(ctx, "generate info event record failed", "error", err) + return nil, err + } + return eventRecord, nil +} + +func handleWarningAction(ctx context.Context, eventName string, ops ...EventOption) (*EventRecord, error) { + logger.Info(ctx, "trigger warning event", "event_name", eventName) + eventRecord, err := NewWarnPlatformSoftRecord(eventName, ops...) + if err != nil { + logger.Error(ctx, "generate warning event record failed", "error", err) + return nil, err + } + return eventRecord, nil +} + +func handleErrorAction(ctx context.Context, eventName string, ops ...EventOption) (*EventRecord, error) { + logger.Info(ctx, "trigger error event", "event_name", eventName) + eventRecord, err := NewCriticalPlatformSoftRecord(eventName, ops...) + if err != nil { + logger.Error(ctx, "generate error event record failed", "error", err) + return nil, err + } + return eventRecord, nil +} + +func handleCriticalAction(ctx context.Context, content string, ops ...EventOption) (*EventRecord, error) { + // 实际执行发送警告、记录日志等操作 + actionParams := content + // ... logic to send critical level event using actionParams ... + logger.Warn(ctx, "trigger critical event", "message", actionParams) + return nil, nil +} + +func handleExceptionAction(ctx context.Context, content string, ops ...EventOption) (*EventRecord, error) { + // 实际执行发送警告、记录日志等操作 + actionParams := content + // ... logic to send except level event using actionParams ... + logger.Warn(ctx, "trigger except event", "message", actionParams) + return nil, nil +} diff --git a/mq/event/event_options.go b/mq/event/event_options.go new file mode 100644 index 0000000..38e7b45 --- /dev/null +++ b/mq/event/event_options.go @@ -0,0 +1,85 @@ +// Package event define real time data evnet operation functions +package event + +import ( + "maps" + "strings" +) + +// EventOption define option function type for event record creation +type EventOption func(*EventRecord) + +// WithCondition define option function to set event condition description +func WithCondition(cond map[string]any) EventOption { + return func(e *EventRecord) { + if cond != nil { + e.Condition = cond + } + } +} + +// WithSubscriptions define option function to set event attached subscription information +func WithSubscriptions(subs []any) EventOption { + return func(e *EventRecord) { + if subs != nil { + e.AttachedSubscriptions = subs + } + } +} + +// WithOperations define option function to set event operation records +func WithOperations(ops []OperationRecord) EventOption { + return func(e *EventRecord) { + if ops != nil { + e.Operations = ops + } + } +} + +// WithCategory define option function to set event category +func WithCategory(cat string) EventOption { + return func(e *EventRecord) { + e.Category = cat + } +} + +// WithResult define option function to set event analysis result +func WithResult(result map[string]any) EventOption { + return func(e *EventRecord) { + e.Result = result + } +} + +func WithTEAnalysisResult(breachType string) EventOption { + return func(e *EventRecord) { + if e.Result == nil { + e.Result = make(map[string]any) + } + + description := "数据异常" + switch strings.ToLower(breachType) { + case "upup": + description = "超越上上限" + case "up": + description = "超越上限" + case "down": + description = "超越下限" + case "downdown": + description = "超越下下限" + } + + e.Result["analysis_desc"] = description + e.Result["breach_type"] = breachType + } +} + +// WithConditionValue define option function to set event condition with real time value and extra data +func WithConditionValue(realTimeValue []float64, extraData map[string]any) EventOption { + return func(e *EventRecord) { + if e.Condition == nil { + e.Condition = make(map[string]any) + } + e.Condition["real_time_value"] = realTimeValue + maps.Copy(e.Condition, extraData) + } +} diff --git a/mq/event/gen_event.go b/mq/event/gen_event.go new file mode 100644 index 0000000..7b4b094 --- /dev/null +++ b/mq/event/gen_event.go @@ -0,0 +1,68 @@ +// Package event define real time data evnet operation functions +package event + +import ( + "fmt" + "time" + + "modelRT/constants" + + "github.com/gofrs/uuid" +) + +// NewPlatformEventRecord define func to create a new platform event record with common fields initialized +func NewPlatformEventRecord(eventType int, priority int, eventName string, opts ...EventOption) (*EventRecord, error) { + u, err := uuid.NewV4() + if err != nil { + return nil, fmt.Errorf("failed to generate UUID: %w", err) + } + + record := &EventRecord{ + EventName: eventName, + EventUUID: u.String(), + Type: eventType, + Priority: priority, + Status: 1, + From: constants.EventFromPlatform, + Timestamp: time.Now().UnixNano() / int64(time.Millisecond), + Condition: make(map[string]any), + AttachedSubscriptions: make([]any, 0), + Operations: make([]OperationRecord, 0), + } + + for _, opt := range opts { + opt(record) + } + + return record, nil +} + +// NewGeneralPlatformSoftRecord define func to create a new general platform software event record +func NewGeneralPlatformSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventGeneralPlatformSoft), 0, name, opts...) +} + +// NewGeneralApplicationSoftRecord define func to create a new general application software event record +func NewGeneralApplicationSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventGeneralApplicationSoft), 0, name, opts...) +} + +// NewWarnPlatformSoftRecord define func to create a new warning platform software event record +func NewWarnPlatformSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventWarnPlatformSoft), 3, name, opts...) +} + +// NewWarnApplicationSoftRecord define func to create a new warning application software event record +func NewWarnApplicationSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventWarnApplicationSoft), 3, name, opts...) +} + +// NewCriticalPlatformSoftRecord define func to create a new critical platform software event record +func NewCriticalPlatformSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventCriticalPlatformSoft), 6, name, opts...) +} + +// NewCriticalApplicationSoftRecord define func to create a new critical application software event record +func NewCriticalApplicationSoftRecord(name string, opts ...EventOption) (*EventRecord, error) { + return NewPlatformEventRecord(int(constants.EventCriticalApplicationSoft), 6, name, opts...) +} diff --git a/mq/event/task_event_gen.go b/mq/event/task_event_gen.go new file mode 100644 index 0000000..ed32c4f --- /dev/null +++ b/mq/event/task_event_gen.go @@ -0,0 +1,81 @@ +// Package event define real time data evnet operation functions +package event + +import ( + "modelRT/constants" +) + +// NewTaskSubmittedMessage creates a message record for when a task is submitted to the queue +func NewTaskSubmittedMessage(taskID, taskType string, priority int) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_submitted", + WithCategory(constants.MessageTaskSubmittedCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + "priority": priority, + }), + ) +} + +// NewTaskRunningMessage creates a message record for when a task begins execution +func NewTaskRunningMessage(taskID, taskType string) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_running", + WithCategory(constants.MessageTaskRunningCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + }), + ) +} + +// NewTaskCompletedMessage creates a message record for when a task finishes successfully +func NewTaskCompletedMessage(taskID, taskType string, executionMs int64) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_completed", + WithCategory(constants.MessageTaskCompletedCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + }), + WithResult(map[string]any{ + "execution_ms": executionMs, + }), + ) +} + +// NewTaskFailedMessage creates a message record for when a task fails during execution +func NewTaskFailedMessage(taskID, taskType, reason string) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_failed", + WithCategory(constants.MessageTaskFailedCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + "reason": reason, + }), + ) +} + +// NewTaskCancelledMessage creates a message record for when a task is cancelled by a user +func NewTaskCancelledMessage(taskID, taskType string) (*EventRecord, error) { + return NewPlatformEventRecord( + int(constants.EventGeneralPlatformSoft), + 0, + "async_task_cancelled", + WithCategory(constants.MessageTaskCancelledCategory), + WithCondition(map[string]any{ + "task_id": taskID, + "task_type": taskType, + }), + ) +} diff --git a/mq/publish_message.go b/mq/publish_message.go new file mode 100644 index 0000000..46e796c --- /dev/null +++ b/mq/publish_message.go @@ -0,0 +1,149 @@ +// Package mq provides read or write access to message queue services +package mq + +import ( + "context" + "encoding/json" + "time" + + "modelRT/constants" + "modelRT/logger" + + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +// MessageMsgChan buffers message records to be published to the message exchange asynchronously +var MessageMsgChan chan *EventMessage + +func init() { + MessageMsgChan = make(chan *EventMessage, 10000) +} + +func initMessageChannel(ctx context.Context) (*amqp.Channel, error) { + channel, err := GetConn().Channel() + if err != nil { + logger.Error(ctx, "open rabbitMQ server channel failed", "error", err) + return nil, err + } + + err = channel.ExchangeDeclare(constants.MessageDeadExchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare message dead letter exchange failed", "error", err) + return nil, err + } + + _, err = channel.QueueDeclare(constants.MessageDeadQueueName, true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare message dead letter queue failed", "error", err) + return nil, err + } + + err = channel.QueueBind(constants.MessageDeadQueueName, constants.MessageDeadRoutingKey, constants.MessageDeadExchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind message dead letter queue failed", "error", err) + return nil, err + } + + err = channel.ExchangeDeclare(constants.MessageExchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare message exchange failed", "error", err) + return nil, err + } + + args := amqp.Table{ + "x-dead-letter-exchange": constants.MessageDeadExchangeName, + "x-dead-letter-routing-key": constants.MessageDeadRoutingKey, + } + _, err = channel.QueueDeclare(constants.MessageQueueName, true, false, false, false, args) + if err != nil { + logger.Error(ctx, "declare message queue failed", "error", err) + return nil, err + } + + err = channel.QueueBind(constants.MessageQueueName, constants.MessageRoutingKey, constants.MessageExchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind message queue failed", "error", err) + return nil, err + } + + if err := channel.Confirm(false); err != nil { + logger.Error(ctx, "channel could not be put into confirm mode", "error", err) + return nil, err + } + return channel, nil +} + +// PushMessageToRabbitMQ publishes message records from msgChan to the message exchange. +// The category of each record is used as the routing key so consumers can bind selectively. +func PushMessageToRabbitMQ(ctx context.Context, msgChan chan *EventMessage) { + channel, err := initMessageChannel(ctx) + if err != nil { + logger.Error(ctx, "initializing message rabbitMQ channel failed", "error", err) + return + } + + confirms := channel.NotifyPublish(make(chan amqp.Confirmation, 100)) + + go func() { + for { + select { + case confirm, ok := <-confirms: + if !ok { + return + } + if !confirm.Ack { + logger.Error(ctx, "publish message failed (rejected by rabbitMQ)", "tag", confirm.DeliveryTag) + } + case <-ctx.Done(): + return + } + } + }() + + for { + select { + case <-ctx.Done(): + logger.Info(ctx, "push message to rabbitMQ stopped by context cancel") + channel.Close() + return + case msg, ok := <-msgChan: + if !ok { + logger.Info(ctx, "push message to rabbitMQ stopped by msgChan closed") + channel.Close() + return + } + + record := msg.Record + + recordBytes, err := json.Marshal(record) + if err != nil { + logger.Error(ctx, "marshal message record failed", "event_uuid", record.EventUUID, "error", err) + continue + } + + msgCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.TraceCarrier)) + headers := amqp.Table{} + otel.GetTextMapPropagator().Inject(msgCtx, amqpHeaderCarrier(headers)) + + routingKey := record.Category + pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + err = channel.PublishWithContext(pubCtx, + constants.MessageExchangeName, + routingKey, + false, + false, + amqp.Publishing{ + ContentType: "text/plain", + Body: recordBytes, + Headers: headers, + }) + cancel() + + if err != nil { + logger.Error(ctx, "publish message to rabbitMQ failed", "routing_key", routingKey, "error", err) + } + } + } +} diff --git a/mq/publish_up_down_limit_event.go b/mq/publish_up_down_limit_event.go new file mode 100644 index 0000000..07a122b --- /dev/null +++ b/mq/publish_up_down_limit_event.go @@ -0,0 +1,188 @@ +// Package mq provides read or write access to message queue services +package mq + +import ( + "context" + "encoding/json" + "time" + + "modelRT/constants" + "modelRT/logger" + "modelRT/mq/event" + + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +// amqpHeaderCarrier adapts amqp.Table to propagation.TextMapCarrier for trace context injection. +type amqpHeaderCarrier amqp.Table + +func (c amqpHeaderCarrier) Get(key string) string { + val, ok := amqp.Table(c)[key] + if !ok { + return "" + } + str, _ := val.(string) + return str +} + +func (c amqpHeaderCarrier) Set(key, value string) { + amqp.Table(c)[key] = value +} + +func (c amqpHeaderCarrier) Keys() []string { + keys := make([]string, 0, len(c)) + for k := range c { + keys = append(keys, k) + } + return keys +} + +var _ propagation.TextMapCarrier = amqpHeaderCarrier{} + +// EventMessage wraps an EventRecord with the trace context of the computation cycle that produced it. +type EventMessage struct { + Record *event.EventRecord + TraceCarrier map[string]string +} + +// MsgChan define variable of channel to store messages that need to be sent to rabbitMQ +var MsgChan chan *EventMessage + +func init() { + MsgChan = make(chan *EventMessage, 10000) +} + +func initUpDownLimitEventChannel(ctx context.Context) (*amqp.Channel, error) { + var channel *amqp.Channel + var err error + + channel, err = GetConn().Channel() + if err != nil { + logger.Error(ctx, "open rabbitMQ server channel failed", "error", err) + return nil, err + } + + err = channel.ExchangeDeclare(constants.EventDeadExchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare event dead letter exchange failed", "error", err) + return nil, err + } + + _, err = channel.QueueDeclare(constants.EventUpDownDeadQueueName, true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare event dead letter queue failed", "error", err) + return nil, err + } + + err = channel.QueueBind(constants.EventUpDownDeadQueueName, "#", constants.EventDeadExchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind event dead letter queue with routing key and exchange failed", "error", err) + return nil, err + } + + err = channel.ExchangeDeclare(constants.EventExchangeName, "topic", true, false, false, false, nil) + if err != nil { + logger.Error(ctx, "declare event exchange failed", "error", err) + return nil, err + } + + args := amqp.Table{ + "x-max-length": int32(50), + "x-dead-letter-exchange": constants.EventDeadExchangeName, + "x-dead-letter-routing-key": constants.EventUpDownDeadRoutingKey, + } + _, err = channel.QueueDeclare(constants.EventUpDownQueueName, true, false, false, false, args) + if err != nil { + logger.Error(ctx, "declare event queue failed", "error", err) + return nil, err + } + + err = channel.QueueBind(constants.EventUpDownQueueName, constants.EventUpDownRoutingKey, constants.EventExchangeName, false, nil) + if err != nil { + logger.Error(ctx, "bind event queue with routing key and exchange failed", "error", err) + return nil, err + } + + if err := channel.Confirm(false); err != nil { + logger.Error(ctx, "channel could not be put into confirm mode", "error", err) + return nil, err + } + return channel, nil +} + +// PushUpDownLimitEventToRabbitMQ define func to push up and down limit event message to rabbitMQ +func PushUpDownLimitEventToRabbitMQ(ctx context.Context, msgChan chan *EventMessage) { + channel, err := initUpDownLimitEventChannel(ctx) + if err != nil { + logger.Error(ctx, "initializing rabbitMQ channel failed", "error", err) + return + } + // TODO 使用配置修改确认模式通道参数 + confirms := channel.NotifyPublish(make(chan amqp.Confirmation, 100)) + + go func() { + for { + select { + case confirm, ok := <-confirms: + if !ok { + return + } + if !confirm.Ack { + logger.Error(ctx, "publish message failed (rejected by rabbitMQ)", "tag", confirm.DeliveryTag) + } + case <-ctx.Done(): + return + } + } + }() + + for { + select { + case <-ctx.Done(): + logger.Info(ctx, "push event alarm message to rabbitMQ stopped by context cancel") + channel.Close() + return + case msg, ok := <-msgChan: + if !ok { + logger.Info(ctx, "push event alarm message to rabbitMQ stopped by msgChan closed, exiting push loop") + channel.Close() + return + } + + eventRecord := msg.Record + + // TODO 将消息的序列化移动到发送之前，以便使用eventRecord的category来作为routing key + recordBytes, err := json.Marshal(eventRecord) + if err != nil { + logger.Error(ctx, "marshal event record failed", "event_uuid", eventRecord.EventUUID, "error", err) + continue + } + + // restore computation cycle trace context so the AMQP message carries the correct parent span + msgCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(msg.TraceCarrier)) + headers := amqp.Table{} + otel.GetTextMapPropagator().Inject(msgCtx, amqpHeaderCarrier(headers)) + + // send event alarm message to rabbitMQ queue + routingKey := eventRecord.Category + pubCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + err = channel.PublishWithContext(pubCtx, + constants.EventExchangeName, // exchange + routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + ContentType: "text/plain", + Body: recordBytes, + Headers: headers, + }) + cancel() + + if err != nil { + logger.Error(ctx, "publish message to rabbitMQ queue failed", "message", recordBytes, "error", err) + } + } + } +} diff --git a/mq/rabbitmq_init.go b/mq/rabbitmq_init.go new file mode 100644 index 0000000..96c4fc0 --- /dev/null +++ b/mq/rabbitmq_init.go @@ -0,0 +1,217 @@ +// Package mq define message queue operation functions +package mq + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "sync" + "time" + + "modelRT/config" + "modelRT/logger" + + amqp "github.com/rabbitmq/amqp091-go" + "github.com/youmark/pkcs8" +) + +var ( + _globalRabbitMQProxy *RabbitMQProxy + rabbitMQOnce sync.Once +) + +// RabbitMQProxy define stuct of rabbitMQ connection proxy +type RabbitMQProxy struct { + tlsConf *tls.Config + conn *amqp.Connection + cancel context.CancelFunc + mu sync.Mutex +} + +// rabbitMQCertConf define stuct of rabbitMQ connection certificates config +type rabbitMQCertConf struct { + serverName string + insecureSkipVerify bool + clientCert tls.Certificate + caCertPool *x509.CertPool +} + +// GetConn define func to return the rabbitMQ connection +func GetConn() *amqp.Connection { + _globalRabbitMQProxy.mu.Lock() + defer _globalRabbitMQProxy.mu.Unlock() + return _globalRabbitMQProxy.conn +} + +// InitRabbitProxy return instance of rabbitMQ connection +func InitRabbitProxy(ctx context.Context, rCfg config.RabbitMQConfig) *RabbitMQProxy { + amqpURI := generateRabbitMQURI(rCfg) + tlsConf, err := initCertConf(rCfg) + if err != nil { + logger.Error(ctx, "init rabbitMQ cert config failed", "error", err) + panic(err) + } + rabbitMQOnce.Do(func() { + cancelCtx, cancel := context.WithCancel(ctx) + conn := initRabbitMQ(ctx, amqpURI, tlsConf) + _globalRabbitMQProxy = &RabbitMQProxy{tlsConf: tlsConf, conn: conn, cancel: cancel} + go _globalRabbitMQProxy.handleReconnect(cancelCtx, amqpURI) + }) + return _globalRabbitMQProxy +} + +// initRabbitMQ return instance of rabbitMQ connection +func initRabbitMQ(ctx context.Context, rabbitMQURI string, tlsConf *tls.Config) *amqp.Connection { + logger.Info(ctx, "connecting to rabbitMQ server", "rabbitmq_uri", rabbitMQURI) + conn, err := amqp.DialConfig(rabbitMQURI, amqp.Config{ + TLSClientConfig: tlsConf, + SASL: []amqp.Authentication{&amqp.ExternalAuth{}}, + Heartbeat: 10 * time.Second, + }) + if err != nil { + logger.Error(ctx, "init rabbitMQ connection failed", "error", err) + panic(err) + } + + return conn +} + +func (p *RabbitMQProxy) handleReconnect(ctx context.Context, rabbitMQURI string) { + for { + closeChan := make(chan *amqp.Error) + GetConn().NotifyClose(closeChan) + + select { + case <-ctx.Done(): + logger.Info(ctx, "context cancelled, exiting handleReconnect") + return + case err, ok := <-closeChan: + if !ok { + logger.Info(ctx, "rabbitMQ notify channel closed") + return + } + + if err == nil { + logger.Info(ctx, "rabbitMQ connection closed normally, no need to reconnect") + return + } + + logger.Warn(ctx, "rabbitMQ connection closed by error, starting reconnect", "reason", err) + } + + if !p.reconnect(ctx, rabbitMQURI) { + return + } + } +} + +func (p *RabbitMQProxy) reconnect(ctx context.Context, rabbitMQURI string) bool { + for { + logger.Info(ctx, "attempting to reconnect to rabbitMQ...") + select { + case <-ctx.Done(): + return false + case <-time.After(5 * time.Second): + + } + + newConn, err := amqp.DialConfig(rabbitMQURI, amqp.Config{ + TLSClientConfig: p.tlsConf, + SASL: []amqp.Authentication{&amqp.ExternalAuth{}}, + Heartbeat: 10 * time.Second, + }) + if err == nil { + p.mu.Lock() + p.conn = newConn + p.mu.Unlock() + logger.Info(ctx, "rabbitMQ reconnected successfully") + return true + } + logger.Error(ctx, "rabbitMQ reconnect failed, will retry", "err", err) + } +} + +// CloseRabbitProxy close the rabbitMQ connection and stop reconnect goroutine +func CloseRabbitProxy() { + if _globalRabbitMQProxy != nil { + _globalRabbitMQProxy.cancel() + _globalRabbitMQProxy.mu.Lock() + if _globalRabbitMQProxy.conn != nil { + _globalRabbitMQProxy.conn.Close() + } + _globalRabbitMQProxy.mu.Unlock() + } +} + +func generateRabbitMQURI(rCfg config.RabbitMQConfig) string { + // TODO 考虑拆分用户名密码配置项，兼容不同认证方式 + // user := url.QueryEscape(rCfg.User) + // password := url.QueryEscape(rCfg.Password) + + // amqpURI := fmt.Sprintf("amqps://%s:%s@%s:%d/", + // user, + // password, + // rCfg.Host, + // rCfg.Port, + // ) + amqpURI := fmt.Sprintf("amqps://%s:%d/", + rCfg.Host, + rCfg.Port, + ) + return amqpURI +} + +func initCertConf(rCfg config.RabbitMQConfig) (*tls.Config, error) { + tlsConf := &tls.Config{ + InsecureSkipVerify: rCfg.InsecureSkipVerify, + ServerName: rCfg.ServerName, + } + + caCert, err := os.ReadFile(rCfg.CACertPath) + if err != nil { + return nil, fmt.Errorf("read server ca file failed: %w", err) + } + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + return nil, fmt.Errorf("failed to parse root certificate from %s", rCfg.CACertPath) + } + tlsConf.RootCAs = caCertPool + + certPEM, err := os.ReadFile(rCfg.ClientCertPath) + if err != nil { + return nil, fmt.Errorf("read client cert file failed: %w", err) + } + + keyData, err := os.ReadFile(rCfg.ClientKeyPath) + if err != nil { + return nil, fmt.Errorf("read private key file failed: %w", err) + } + + block, _ := pem.Decode(keyData) + if block == nil { + return nil, fmt.Errorf("failed to decode PEM block from private key") + } + + der, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(rCfg.ClientKeyPassword)) + if err != nil { + return nil, fmt.Errorf("parse password-protected private key failed: %w", err) + } + + privBytes, err := x509.MarshalPKCS8PrivateKey(der) + if err != nil { + return nil, fmt.Errorf("marshal private key failed: %w", err) + } + + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + + clientCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, fmt.Errorf("create x509 key pair failed: %w", err) + } + + tlsConf.Certificates = []tls.Certificate{clientCert} + return tlsConf, nil +} diff --git a/network/api_endpoint.go b/network/api_endpoint.go index 6f15665..cac8d2a 100644 --- a/network/api_endpoint.go +++ b/network/api_endpoint.go @@ -77,7 +77,7 @@ func PollAPIEndpoints(endpoint APIEndpoint) ([]float64, error) { } dataLen := len(realDataJSON.Get("data").MustArray()) - for i := 0; i < dataLen; i++ { + for i := range dataLen { valueSlice = append(valueSlice, realDataJSON.Get("data").GetIndex(i).Get("value").MustFloat64()) } return valueSlice, nil diff --git a/network/async_task_request.go b/network/async_task_request.go new file mode 100644 index 0000000..8982ef6 --- /dev/null +++ b/network/async_task_request.go @@ -0,0 +1,96 @@ +// Package network define struct of network operation +package network + +import ( + "time" + + "github.com/gofrs/uuid" +) + +// AsyncTaskCreateRequest defines the request structure for creating an asynchronous task +type AsyncTaskCreateRequest struct { + // required: true + // enum: TOPOLOGY_ANALYSIS, PERFORMANCE_ANALYSIS, EVENT_ANALYSIS, BATCH_IMPORT + TaskType string `json:"task_type" example:"TOPOLOGY_ANALYSIS" description:"异步任务类型"` + // required: true + Params map[string]any `json:"params" swaggertype:"object" description:"任务参数，根据任务类型不同而不同"` +} + +// AsyncTaskCreateResponse defines the response structure for creating an asynchronous task +type AsyncTaskCreateResponse struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` +} + +// AsyncTaskResultQueryRequest defines the request structure for querying task results +type AsyncTaskResultQueryRequest struct { + // required: true + TaskIDs []uuid.UUID `json:"task_ids" swaggertype:"array,string" example:"[\"123e4567-e89b-12d3-a456-426614174000\",\"223e4567-e89b-12d3-a456-426614174001\"]" description:"任务ID列表"` +} + +// AsyncTaskResult defines the structure for a single task result +type AsyncTaskResult struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` + TaskType string `json:"task_type" example:"TOPOLOGY_ANALYSIS" description:"任务类型"` + Status string `json:"status" example:"COMPLETED" description:"任务状态：SUBMITTED, RUNNING, COMPLETED, FAILED"` + Progress *int `json:"progress,omitempty" example:"65" description:"任务进度(0-100)，仅当状态为RUNNING时返回"` + CreatedAt int64 `json:"created_at" example:"1741846200" description:"任务创建时间戳"` + FinishedAt *int64 `json:"finished_at,omitempty" example:"1741846205" description:"任务完成时间戳，仅当状态为COMPLETED或FAILED时返回"` + Result map[string]any `json:"result,omitempty" swaggertype:"object" description:"任务结果，仅当状态为COMPLETED时返回"` + ErrorCode *int `json:"error_code,omitempty" example:"400102" description:"错误码，仅当状态为FAILED时返回"` + ErrorMessage *string `json:"error_message,omitempty" example:"Component UUID not found" description:"错误信息，仅当状态为FAILED时返回"` + ErrorDetail map[string]any `json:"error_detail,omitempty" swaggertype:"object" description:"错误详情，仅当状态为FAILED时返回"` +} + +// AsyncTaskResultQueryResponse defines the response structure for querying task results +type AsyncTaskResultQueryResponse struct { + Total int `json:"total" example:"3" description:"查询的任务总数"` + Tasks []AsyncTaskResult `json:"tasks" description:"任务结果列表"` +} + +// AsyncTaskProgressUpdate defines the structure for task progress update +type AsyncTaskProgressUpdate struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` + Progress int `json:"progress" example:"50" description:"任务进度(0-100)"` +} + +// AsyncTaskStatusUpdate defines the structure for task status update +type AsyncTaskStatusUpdate struct { + TaskID uuid.UUID `json:"task_id" example:"123e4567-e89b-12d3-a456-426614174000" description:"任务唯一标识符"` + Status string `json:"status" example:"RUNNING" description:"任务状态：SUBMITTED, RUNNING, COMPLETED, FAILED"` + Timestamp int64 `json:"timestamp" example:"1741846205" description:"状态更新时间戳"` +} + +// TopologyAnalysisParams defines the parameters for topology analysis task +type TopologyAnalysisParams struct { + StartComponentUUID string `json:"start_component_uuid" example:"550e8400-e29b-41d4-a716-446655440000" description:"起始元件UUID"` + EndComponentUUID string `json:"end_component_uuid" example:"550e8400-e29b-41d4-a716-446655440001" description:"目标元件UUID"` + CheckInService bool `json:"check_in_service" example:"true" description:"是否检查路径上元件的投运状态，默认为true"` +} + +// PerformanceAnalysisParams defines the parameters for performance analysis task +type PerformanceAnalysisParams struct { + ComponentIDs []string `json:"component_ids" example:"[\"comp-001\",\"comp-002\"]" description:"需要分析的元件ID列表"` + TimeRange struct { + Start time.Time `json:"start" example:"2026-03-01T00:00:00Z" description:"分析开始时间"` + End time.Time `json:"end" example:"2026-03-02T00:00:00Z" description:"分析结束时间"` + } `json:"time_range" description:"分析时间范围"` +} + +// EventAnalysisParams defines the parameters for event analysis task +type EventAnalysisParams struct { + EventType string `json:"event_type" example:"MOTOR_START" description:"事件类型"` + StartTime time.Time `json:"start_time" example:"2026-03-01T00:00:00Z" description:"事件开始时间"` + EndTime time.Time `json:"end_time" example:"2026-03-02T00:00:00Z" description:"事件结束时间"` + Components []string `json:"components,omitempty" example:"[\"comp-001\",\"comp-002\"]" description:"关联的元件列表"` +} + +// BatchImportParams defines the parameters for batch import task +type BatchImportParams struct { + FilePath string `json:"file_path" example:"/data/import/model.csv" description:"导入文件路径"` + FileType string `json:"file_type" example:"CSV" description:"文件类型：CSV, JSON, XML"` + Options struct { + Overwrite bool `json:"overwrite" example:"false" description:"是否覆盖现有数据"` + Validate bool `json:"validate" example:"true" description:"是否进行数据验证"` + NotifyUser bool `json:"notify_user" example:"true" description:"是否通知用户"` + } `json:"options" description:"导入选项"` +} diff --git a/network/attr_request.go b/network/attr_request.go index fad5edd..624f5f9 100644 --- a/network/attr_request.go +++ b/network/attr_request.go @@ -8,8 +8,8 @@ type AttrGetRequest struct { // AttrSetRequest defines the request payload for setting an attribute type AttrSetRequest struct { - AttrToken string `json:"attr_token"` - AttrValue interface{} `json:"attr_value"` + AttrToken string `json:"attr_token"` + AttrValue any `json:"attr_value"` } // AttrDeleteRequest defines the request payload for deleting an attribute diff --git a/network/circuit_diagram_update_request.go b/network/circuit_diagram_update_request.go index 06966cd..76ed6c2 100644 --- a/network/circuit_diagram_update_request.go +++ b/network/circuit_diagram_update_request.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "modelRT/common" "modelRT/common/errcode" "modelRT/constants" "modelRT/orm" @@ -64,10 +65,10 @@ func ParseUUID(info TopologicChangeInfo) (TopologicUUIDChangeInfos, error) { switch info.ChangeType { case constants.UUIDFromChangeType: if info.NewUUIDFrom == info.OldUUIDFrom { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDFromCheckT1) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDFromCheckT1) } if info.NewUUIDTo != info.OldUUIDTo { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDToCheckT1) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDToCheckT1) } oldUUIDFrom, err := uuid.FromString(info.OldUUIDFrom) @@ -90,10 +91,10 @@ func ParseUUID(info TopologicChangeInfo) (TopologicUUIDChangeInfos, error) { UUIDChangeInfo.NewUUIDTo = OldUUIDTo case constants.UUIDToChangeType: if info.NewUUIDFrom != info.OldUUIDFrom { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDFromCheckT2) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDFromCheckT2) } if info.NewUUIDTo == info.OldUUIDTo { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDToCheckT2) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDToCheckT2) } oldUUIDFrom, err := uuid.FromString(info.OldUUIDFrom) @@ -116,10 +117,10 @@ func ParseUUID(info TopologicChangeInfo) (TopologicUUIDChangeInfos, error) { UUIDChangeInfo.NewUUIDTo = newUUIDTo case constants.UUIDAddChangeType: if info.OldUUIDFrom != "" { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDFromCheckT3) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDFromCheckT3) } if info.OldUUIDTo != "" { - return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", constants.ErrUUIDToCheckT3) + return UUIDChangeInfo, fmt.Errorf("topologic change data check failed:%w", common.ErrUUIDToCheckT3) } newUUIDFrom, err := uuid.FromString(info.NewUUIDFrom) @@ -157,7 +158,7 @@ func ConvertComponentUpdateInfosToComponents(updateInfo ComponentUpdateInfo) (*o // Op: info.Op, // Tag: info.Tag, // 其他字段可根据需要补充 - Ts: time.Now(), + TS: time.Now(), } return component, nil } diff --git a/network/request_convert.go b/network/request_convert.go index 85c9fe0..11ca2d3 100644 --- a/network/request_convert.go +++ b/network/request_convert.go @@ -8,7 +8,7 @@ import ( ) // ConvertAnyComponentInfosToComponents define convert any component request info to component struct -func ConvertAnyComponentInfosToComponents(anyInfo interface{}) (*orm.Component, error) { +func ConvertAnyComponentInfosToComponents(anyInfo any) (*orm.Component, error) { switch info := anyInfo.(type) { case ComponentCreateInfo: return ConvertComponentCreateInfosToComponents(info) diff --git a/network/response.go b/network/response.go index e8d775a..b111df7 100644 --- a/network/response.go +++ b/network/response.go @@ -3,18 +3,25 @@ package network // FailureResponse define struct of standard failure API response format type FailureResponse struct { - Code int `json:"code" example:"500"` - Msg string `json:"msg" example:"failed to get recommend data from redis"` + Code int `json:"code" example:"3000"` + Msg string `json:"msg" example:"process completed with partial failures"` Payload any `json:"payload" swaggertype:"object"` } // SuccessResponse define struct of standard successful API response format type SuccessResponse struct { - Code int `json:"code" example:"200"` - Msg string `json:"msg" example:"success"` + Code int `json:"code" example:"2000"` + Msg string `json:"msg" example:"process completed"` Payload any `json:"payload" swaggertype:"object"` } +// WSResponse define struct of standard websocket API response format +type WSResponse struct { + Code int `json:"code" example:"2000"` + Msg string `json:"msg" example:"process completed"` + Payload any `json:"payload,omitempty" swaggertype:"object"` +} + // MeasurementRecommendPayload define struct of represents the data payload for the successful recommendation response. type MeasurementRecommendPayload struct { Input string `json:"input" example:"transformfeeder1_220."` @@ -26,7 +33,7 @@ type MeasurementRecommendPayload struct { // TargetResult define struct of target item in real time data subscription response payload type TargetResult struct { ID string `json:"id" example:"grid1.zone1.station1.ns1.tag1.transformfeeder1_220.I_A_rms"` - Code string `json:"code" example:"1001"` + Code int `json:"code" example:"20000"` Msg string `json:"msg" example:"subscription success"` } diff --git a/orm/async_motor.go b/orm/async_motor.go index 017e46e..48547ec 100644 --- a/orm/async_motor.go +++ b/orm/async_motor.go @@ -85,7 +85,6 @@ func (a *AsyncMotor) TableName() string { // SetComponentID func implement BasicModelInterface interface func (a *AsyncMotor) SetComponentID(componentID int64) { a.ComponentID = componentID - return } // ReturnTableName func implement BasicModelInterface interface diff --git a/orm/async_task.go b/orm/async_task.go new file mode 100644 index 0000000..37709bf --- /dev/null +++ b/orm/async_task.go @@ -0,0 +1,129 @@ +// Package orm define database data struct +package orm + +import ( + "github.com/gofrs/uuid" +) + +// AsyncTaskType defines the type of asynchronous task +type AsyncTaskType string + +const ( + // AsyncTaskTypeTopologyAnalysis represents topology analysis task + AsyncTaskTypeTopologyAnalysis AsyncTaskType = "TOPOLOGY_ANALYSIS" + // AsyncTaskTypePerformanceAnalysis represents performance analysis task + AsyncTaskTypePerformanceAnalysis AsyncTaskType = "PERFORMANCE_ANALYSIS" + // AsyncTaskTypeEventAnalysis represents event analysis task + AsyncTaskTypeEventAnalysis AsyncTaskType = "EVENT_ANALYSIS" + // AsyncTaskTypeBatchImport represents batch import task + AsyncTaskTypeBatchImport AsyncTaskType = "BATCH_IMPORT" + // AsyncTaskTypeTest represents test task for system verification + AsyncTaskTypeTest AsyncTaskType = "TEST" +) + +// AsyncTaskStatus defines the status of asynchronous task +type AsyncTaskStatus string + +const ( + // AsyncTaskStatusSubmitted represents task has been submitted to queue + AsyncTaskStatusSubmitted AsyncTaskStatus = "SUBMITTED" + // AsyncTaskStatusRunning represents task is currently executing + AsyncTaskStatusRunning AsyncTaskStatus = "RUNNING" + // AsyncTaskStatusCompleted represents task completed successfully + AsyncTaskStatusCompleted AsyncTaskStatus = "COMPLETED" + // AsyncTaskStatusFailed represents task failed with error + AsyncTaskStatusFailed AsyncTaskStatus = "FAILED" +) + +// AsyncTask defines the core task entity stored in database for task lifecycle tracking +type AsyncTask struct { + TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid;default:gen_random_uuid()"` + TaskType AsyncTaskType `gorm:"column:task_type;type:varchar(50);not null;index"` + Status AsyncTaskStatus `gorm:"column:status;type:varchar(20);not null;index"` + Params JSONMap `gorm:"column:params;type:jsonb"` + CreatedAt int64 `gorm:"column:created_at;not null;index"` + FinishedAt *int64 `gorm:"column:finished_at;index"` + StartedAt *int64 `gorm:"column:started_at;index"` + ExecutionTime *int64 `gorm:"column:execution_time"` + Progress *int `gorm:"column:progress"` // 0-100, nullable + RetryCount int `gorm:"column:retry_count;default:0"` + MaxRetryCount int `gorm:"column:max_retry_count;default:3"` + NextRetryTime *int64 `gorm:"column:next_retry_time;index"` + RetryDelay int `gorm:"column:retry_delay;default:5000"` + Priority int `gorm:"column:priority;default:5;index"` + QueueName string `gorm:"column:queue_name;type:varchar(100);default:'default'"` + WorkerID *string `gorm:"column:worker_id;type:varchar(50)"` + FailureReason *string `gorm:"column:failure_reason;type:text"` + StackTrace *string `gorm:"column:stack_trace;type:text"` + CreatedBy *string `gorm:"column:created_by;type:varchar(100)"` +} + +// TableName returns the table name for AsyncTask model +func (a *AsyncTask) TableName() string { + return "async_task" +} + +// SetSubmitted marks the task as submitted +func (a *AsyncTask) SetSubmitted() { + a.Status = AsyncTaskStatusSubmitted +} + +// SetRunning marks the task as running +func (a *AsyncTask) SetRunning() { + a.Status = AsyncTaskStatusRunning +} + +// SetCompleted marks the task as completed with finished timestamp +func (a *AsyncTask) SetCompleted(timestamp int64) { + a.Status = AsyncTaskStatusCompleted + a.FinishedAt = ×tamp + a.setProgress(100) +} + +// SetFailed marks the task as failed with finished timestamp +func (a *AsyncTask) SetFailed(timestamp int64) { + a.Status = AsyncTaskStatusFailed + a.FinishedAt = ×tamp +} + +// setProgress updates the task progress (0-100) +func (a *AsyncTask) setProgress(value int) { + if value < 0 { + value = 0 + } + if value > 100 { + value = 100 + } + a.Progress = &value +} + +// UpdateProgress updates the task progress with validation +func (a *AsyncTask) UpdateProgress(value int) { + a.setProgress(value) +} + +// IsCompleted checks if the task is completed +func (a *AsyncTask) IsCompleted() bool { + return a.Status == AsyncTaskStatusCompleted +} + +// IsRunning checks if the task is running +func (a *AsyncTask) IsRunning() bool { + return a.Status == AsyncTaskStatusRunning +} + +// IsFailed checks if the task failed +func (a *AsyncTask) IsFailed() bool { + return a.Status == AsyncTaskStatusFailed +} + +// IsValidTaskType checks if the task type is valid +func IsValidAsyncTaskType(taskType string) bool { + switch AsyncTaskType(taskType) { + case AsyncTaskTypeTopologyAnalysis, AsyncTaskTypePerformanceAnalysis, + AsyncTaskTypeEventAnalysis, AsyncTaskTypeBatchImport, AsyncTaskTypeTest: + return true + default: + return false + } +} diff --git a/orm/async_task_result.go b/orm/async_task_result.go new file mode 100644 index 0000000..bbe9137 --- /dev/null +++ b/orm/async_task_result.go @@ -0,0 +1,75 @@ +// Package orm define database data struct +package orm + +import ( + "github.com/gofrs/uuid" +) + +// AsyncTaskResult stores computation results, separate from AsyncTask model for flexibility +type AsyncTaskResult struct { + TaskID uuid.UUID `gorm:"column:task_id;primaryKey;type:uuid"` + Result JSONMap `gorm:"column:result;type:jsonb"` + ErrorCode *int `gorm:"column:error_code"` + ErrorMessage *string `gorm:"column:error_message;type:text"` + ErrorDetail JSONMap `gorm:"column:error_detail;type:jsonb"` + ExecutionTime int64 `gorm:"column:execution_time;not null;default:0"` + MemoryUsage *int64 `gorm:"column:memory_usage"` + CPUUsage *float64 `gorm:"column:cpu_usage"` + RetryCount int `gorm:"column:retry_count;default:0"` + CompletedAt int64 `gorm:"column:completed_at;not null"` +} + +// TableName returns the table name for AsyncTaskResult model +func (a *AsyncTaskResult) TableName() string { + return "async_task_result" +} + +// SetSuccess sets the result for successful task execution +func (a *AsyncTaskResult) SetSuccess(result JSONMap) { + a.Result = result + a.ErrorCode = nil + a.ErrorMessage = nil + a.ErrorDetail = nil +} + +// SetError sets the error information for failed task execution +func (a *AsyncTaskResult) SetError(code int, message string, detail JSONMap) { + a.Result = nil + a.ErrorCode = &code + a.ErrorMessage = &message + a.ErrorDetail = detail +} + +// HasError checks if the task result contains an error +func (a *AsyncTaskResult) HasError() bool { + return a.ErrorCode != nil || a.ErrorMessage != nil +} + +// GetErrorCode returns the error code or 0 if no error +func (a *AsyncTaskResult) GetErrorCode() int { + if a.ErrorCode == nil { + return 0 + } + return *a.ErrorCode +} + +// GetErrorMessage returns the error message or empty string if no error +func (a *AsyncTaskResult) GetErrorMessage() string { + if a.ErrorMessage == nil { + return "" + } + return *a.ErrorMessage +} + +// IsSuccess checks if the task execution was successful +func (a *AsyncTaskResult) IsSuccess() bool { + return !a.HasError() +} + +// Clear clears all result data +func (a *AsyncTaskResult) Clear() { + a.Result = nil + a.ErrorCode = nil + a.ErrorMessage = nil + a.ErrorDetail = nil +} diff --git a/orm/busbar_section.go b/orm/busbar_section.go index 764057f..d953d2f 100644 --- a/orm/busbar_section.go +++ b/orm/busbar_section.go @@ -72,7 +72,6 @@ func (b *BusbarSection) TableName() string { // SetComponentID func implement BasicModelInterface interface func (b *BusbarSection) SetComponentID(componentID int64) { b.ComponentID = componentID - return } // ReturnTableName func implement BasicModelInterface interface diff --git a/orm/circuit_diagram_bay.go b/orm/circuit_diagram_bay.go index 4554bc3..4396219 100644 --- a/orm/circuit_diagram_bay.go +++ b/orm/circuit_diagram_bay.go @@ -34,7 +34,7 @@ type Bay struct { DevEtc JSONMap `gorm:"column:dev_etc;type:jsonb;not null;default:'[]'"` Components []uuid.UUID `gorm:"column:components;type:uuid[];not null;default:'{}'"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` } // TableName func respresent return table name of Bay diff --git a/orm/circuit_diagram_component.go b/orm/circuit_diagram_component.go index df41bc5..a292c6a 100644 --- a/orm/circuit_diagram_component.go +++ b/orm/circuit_diagram_component.go @@ -27,7 +27,7 @@ type Component struct { Label JSONMap `gorm:"column:label;type:jsonb;not null;default:'{}'"` Context JSONMap `gorm:"column:context;type:jsonb;not null;default:'{}'"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:current_timestamp;autoCreateTime"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:current_timestamp;autoCreateTime"` } // TableName func respresent return table name of Component diff --git a/orm/circuit_diagram_grid.go b/orm/circuit_diagram_grid.go index 43d90e5..86da8a8 100644 --- a/orm/circuit_diagram_grid.go +++ b/orm/circuit_diagram_grid.go @@ -12,7 +12,7 @@ type Grid struct { Name string `gorm:"column:name"` Description string `gorm:"column:description"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Grid diff --git a/orm/circuit_diagram_measurement.go b/orm/circuit_diagram_measurement.go index 4abc441..e281559 100644 --- a/orm/circuit_diagram_measurement.go +++ b/orm/circuit_diagram_measurement.go @@ -20,7 +20,7 @@ type Measurement struct { BayUUID uuid.UUID `gorm:"column:bay_uuid;type:uuid;not null"` ComponentUUID uuid.UUID `gorm:"column:component_uuid;type:uuid;not null"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` } // TableName func respresent return table name of Measurement diff --git a/orm/circuit_diagram_page.go b/orm/circuit_diagram_page.go index 172a693..2e3c50a 100644 --- a/orm/circuit_diagram_page.go +++ b/orm/circuit_diagram_page.go @@ -12,7 +12,7 @@ type Page struct { Context JSONMap `gorm:"column:context;type:jsonb;default:'{}'"` Description string `gorm:"column:description"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Page diff --git a/orm/circuit_diagram_station.go b/orm/circuit_diagram_station.go index e937257..5059e97 100644 --- a/orm/circuit_diagram_station.go +++ b/orm/circuit_diagram_station.go @@ -14,7 +14,7 @@ type Station struct { Description string `gorm:"column:description"` IsLocal bool `gorm:"column:is_local"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Station diff --git a/orm/circuit_diagram_topologic.go b/orm/circuit_diagram_topologic.go index d85137b..7f444e6 100644 --- a/orm/circuit_diagram_topologic.go +++ b/orm/circuit_diagram_topologic.go @@ -16,7 +16,7 @@ type Topologic struct { Flag int `gorm:"column:flag"` Description string `gorm:"column:description;size:512;not null;default:''"` Op int `gorm:"column:op;not null;default:-1"` - Ts time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` + TS time.Time `gorm:"column:ts;type:timestamptz;not null;default:CURRENT_TIMESTAMP"` } // TableName func respresent return table name of Page diff --git a/orm/circuit_diagram_zone.go b/orm/circuit_diagram_zone.go index f50aaee..85caf39 100644 --- a/orm/circuit_diagram_zone.go +++ b/orm/circuit_diagram_zone.go @@ -13,7 +13,7 @@ type Zone struct { Name string `gorm:"column:name"` Description string `gorm:"column:description"` Op int `gorm:"column:op"` - Ts time.Time `gorm:"column:ts"` + TS time.Time `gorm:"column:ts"` } // TableName func respresent return table name of Zone diff --git a/orm/demo.go b/orm/demo.go index 8b0991b..2483066 100644 --- a/orm/demo.go +++ b/orm/demo.go @@ -14,7 +14,7 @@ type Demo struct { UIAlarm float32 `gorm:"column:ui_alarm" json:"ui_alarm"` // 低电流告警值 OIAlarm float32 `gorm:"column:oi_alarm" json:"oi_alarm"` // 高电流告警值 Op int `gorm:"column:op" json:"op"` // 操作人 ID - Ts time.Time `gorm:"column:ts" json:"ts"` // 操作时间 + TS time.Time `gorm:"column:ts" json:"ts"` // 操作时间 } // TableName func respresent return table name of busbar section @@ -25,7 +25,6 @@ func (d *Demo) TableName() string { // SetComponentID func implement BasicModelInterface interface func (d *Demo) SetComponentID(componentID int64) { d.ComponentID = componentID - return } // ReturnTableName func implement BasicModelInterface interface diff --git a/pool/concurrency_anchor_parse.go b/pool/concurrency_anchor_parse.go index 443f1a7..1e82723 100644 --- a/pool/concurrency_anchor_parse.go +++ b/pool/concurrency_anchor_parse.go @@ -5,11 +5,11 @@ import ( "fmt" "time" - "modelRT/alert" "modelRT/config" "modelRT/constants" "modelRT/diagram" "modelRT/logger" + "modelRT/real-time-data/alert" "github.com/panjf2000/ants/v2" ) @@ -28,7 +28,7 @@ func AnchorPoolInit(concurrentQuantity int) (pool *ants.PoolWithFunc, err error) } // AnchorFunc defines func that process the real time data of component anchor params -var AnchorFunc = func(poolConfig interface{}) { +var AnchorFunc = func(poolConfig any) { var firstStart bool alertManager := alert.GetAlertMangerInstance() diff --git a/pool/concurrency_model_parse.go b/pool/concurrency_model_parse.go index 1fe8718..0872fb0 100644 --- a/pool/concurrency_model_parse.go +++ b/pool/concurrency_model_parse.go @@ -12,7 +12,7 @@ import ( ) // ParseFunc defines func that parses the model data from postgres -var ParseFunc = func(parseConfig interface{}) { +var ParseFunc = func(parseConfig any) { modelParseConfig, ok := parseConfig.(config.ModelParseConfig) if !ok { logger.Error(modelParseConfig.Ctx, "conversion model parse config type failed") diff --git a/alert/init.go b/real-time-data/alert/init.go similarity index 100% rename from alert/init.go rename to real-time-data/alert/init.go diff --git a/real-time-data/cache.go b/real-time-data/cache.go index 70a2d03..585a447 100644 --- a/real-time-data/cache.go +++ b/real-time-data/cache.go @@ -10,7 +10,7 @@ import ( // DataItem define structure for storing data, insertion time, and last access time type DataItem struct { - Data interface{} + Data any InsertTime time.Time LastAccess time.Time Index int @@ -38,14 +38,14 @@ func (pq priorityQueue) Swap(i, j int) { pq[j].item.Index = j } -func (pq *priorityQueue) Push(x interface{}) { +func (pq *priorityQueue) Push(x any) { n := len(*pq) queueItem := x.(*priorityQueueItem) queueItem.item.Index = n *pq = append(*pq, queueItem) } -func (pq *priorityQueue) Pop() interface{} { +func (pq *priorityQueue) Pop() any { old := *pq n := len(old) queueItem := old[n-1] @@ -65,7 +65,7 @@ func (pq *priorityQueue) update(item *DataItem, newPrio int64) { type TimeCache struct { mu sync.Mutex capacity int - items map[interface{}]*DataItem + items map[any]*DataItem pq priorityQueue } @@ -73,13 +73,13 @@ type TimeCache struct { func NewTimeCache(capacity int) *TimeCache { return &TimeCache{ capacity: capacity, - items: make(map[interface{}]*DataItem), + items: make(map[any]*DataItem), pq: make(priorityQueue, 0, capacity), } } // Add 添加一个新项到缓存中 -func (tc *TimeCache) Add(data interface{}) { +func (tc *TimeCache) Add(data any) { tc.mu.Lock() defer tc.mu.Unlock() diff --git a/real-time-data/compute_analyzer.go b/real-time-data/compute_analyzer.go index 1525734..864fd2d 100644 --- a/real-time-data/compute_analyzer.go +++ b/real-time-data/compute_analyzer.go @@ -9,7 +9,11 @@ import ( "modelRT/constants" "modelRT/logger" - "modelRT/real-time-data/event" + "modelRT/mq" + "modelRT/mq/event" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) // RealTimeAnalyzer define interface general methods for real-time data analysis and event triggering @@ -26,6 +30,13 @@ type teEventThresholds struct { isFloatCause bool } +type teBreachTrigger struct { + breachType string + triggered bool + triggeredValues []float64 + eventOpts []event.EventOption +} + // parseTEThresholds define func to parse telemetry thresholds by casue map func parseTEThresholds(cause map[string]any) (teEventThresholds, error) { t := teEventThresholds{} @@ -84,60 +95,77 @@ func (t *TEAnalyzer) AnalyzeAndTriggerEvent(ctx context.Context, conf *ComputeCo // analyzeTEDataLogic define func to processing telemetry data and event triggering func analyzeTEDataLogic(ctx context.Context, conf *ComputeConfig, thresholds teEventThresholds, realTimeValues []float64) { windowSize := conf.minBreachCount - if windowSize <= 0 { - logger.Error(ctx, "variable minBreachCount is invalid or zero, analysis skipped", "minBreachCount", windowSize) + dataLen := len(realTimeValues) + if dataLen < windowSize || windowSize <= 0 { return } - // mark whether any events have been triggered in this batch - var eventTriggered bool - breachTriggers := map[string]bool{ - "up": false, "upup": false, "down": false, "downdown": false, + statusArray := make([]string, dataLen) + for i, val := range realTimeValues { + statusArray[i] = getTEBreachType(val, thresholds) } - // implement slide window to determine breach counts - for i := 0; i <= len(realTimeValues)-windowSize; i++ { - window := realTimeValues[i : i+windowSize] - firstValueBreachType := getTEBreachType(window[0], thresholds) + breachTriggers := make(map[string]teBreachTrigger) + for i := 0; i <= dataLen-windowSize; i++ { + firstBreachType := statusArray[i] - if firstValueBreachType == "" { + // if the first value in the window does not breach, skip this window directly + if firstBreachType == "" { continue } allMatch := true for j := 1; j < windowSize; j++ { - currentValueBreachType := getTEBreachType(window[j], thresholds) - if currentValueBreachType != firstValueBreachType { + if statusArray[i+j] != firstBreachType { allMatch = false break } } if allMatch { + triggerValues := realTimeValues[i : i+windowSize] // in the case of a continuous sequence of out-of-limit events, check whether this type of event has already been triggered in the current batch of data - if !breachTriggers[firstValueBreachType] { - // trigger event - logger.Warn(ctx, "event triggered by sliding window", "breach_type", firstValueBreachType, "value", window[windowSize-1]) + _, exists := breachTriggers[firstBreachType] + if !exists { + logger.Warn(ctx, "event triggered by sliding window", + "breach_type", firstBreachType, + "trigger_values", triggerValues) - breachTriggers[firstValueBreachType] = true - eventTriggered = true + // build Options + opts := []event.EventOption{ + event.WithConditionValue(triggerValues, conf.Cause), + event.WithTEAnalysisResult(firstBreachType), + event.WithCategory(constants.EventWarnUpDownLimitCategroy), + // TODO 生成 operations并考虑如何放入 event 中 + // event.WithOperations(nil) + } + breachTriggers[firstBreachType] = teBreachTrigger{ + breachType: firstBreachType, + triggered: false, + triggeredValues: triggerValues, + eventOpts: opts, + } } } } - if eventTriggered { - command, content := genTEEventCommandAndContent(ctx, conf.Action) - // TODO 考虑 content 是否可以为空,先期不允许 - if command == "" || content == "" { - logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "content", content) + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + + for breachType, trigger := range breachTriggers { + // trigger Action + command, mainBody := genTEEventCommandAndMainBody(ctx, conf.Action) + eventName := fmt.Sprintf("telemetry_%s_%s_Breach_Event", mainBody, breachType) + eventRecord, err := event.TriggerEventAction(ctx, command, eventName, trigger.eventOpts...) + if err != nil { + logger.Error(ctx, "trigger event action failed", "error", err) return } - event.TriggerEventAction(ctx, command, content) - return + mq.MsgChan <- &mq.EventMessage{Record: eventRecord, TraceCarrier: carrier} } } -func genTEEventCommandAndContent(ctx context.Context, action map[string]any) (command string, content string) { +func genTEEventCommandAndMainBody(ctx context.Context, action map[string]any) (command string, mainBody string) { cmdValue, exist := action["command"] if !exist { logger.Error(ctx, "can not find command variable into action map", "action", action) @@ -185,7 +213,7 @@ type tiEventThresholds struct { isFloatCause bool } -// parseTEThresholds define func to parse telesignal thresholds by casue map +// parseTIThresholds define func to parse telesignal thresholds by casue map func parseTIThresholds(cause map[string]any) (tiEventThresholds, error) { edgeKey := "edge" t := tiEventThresholds{ @@ -211,11 +239,12 @@ func parseTIThresholds(cause map[string]any) (tiEventThresholds, error) { // getTIBreachType define func to determine which type of out-of-limit the telesignal real time data belongs to func getTIBreachType(currentValue float64, previousValue float64, t tiEventThresholds) string { - if t.edge == constants.TelesignalRaising { + switch t.edge { + case constants.TelesignalRaising: if previousValue == 0.0 && currentValue == 1.0 { return constants.TIBreachTriggerType } - } else if t.edge == constants.TelesignalFalling { + case constants.TelesignalFalling: if previousValue == 1.0 && currentValue == 0.0 { return constants.TIBreachTriggerType } @@ -297,18 +326,24 @@ func analyzeTIDataLogic(ctx context.Context, conf *ComputeConfig, thresholds tiE } if eventTriggered { - command, content := genTIEventCommandAndContent(conf.Action) - // TODO 考虑 content 是否可以为空,先期不允许 - if command == "" || content == "" { - logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "content", content) + command, mainBody := genTIEventCommandAndMainBody(conf.Action) + if command == "" || mainBody == "" { + logger.Error(ctx, "generate telemetry evnet command or content failed", "action", conf.Action, "command", command, "main_body", mainBody) return } - event.TriggerEventAction(ctx, command, content) + eventRecord, err := event.TriggerEventAction(ctx, command, mainBody) + if err != nil { + logger.Error(ctx, "trigger event action failed", "error", err) + return + } + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + mq.MsgChan <- &mq.EventMessage{Record: eventRecord, TraceCarrier: carrier} return } } -func genTIEventCommandAndContent(action map[string]any) (command string, content string) { +func genTIEventCommandAndMainBody(action map[string]any) (command string, mainBody string) { cmdValue, exist := action["command"] if !exist { return "", "" diff --git a/real-time-data/event/event_handlers.go b/real-time-data/event/event_handlers.go deleted file mode 100644 index a161a7d..0000000 --- a/real-time-data/event/event_handlers.go +++ /dev/null @@ -1,74 +0,0 @@ -// Package event define real time data evnet operation functions -package event - -import ( - "context" - - "modelRT/logger" -) - -type actionHandler func(ctx context.Context, content string) error - -// actionDispatchMap define variable to store all action handler into map -var actionDispatchMap = map[string]actionHandler{ - "info": handleInfoAction, - "warning": handleWarningAction, - "error": handleErrorAction, - "critical": handleCriticalAction, - "exception": handleExceptionAction, -} - -// TriggerEventAction define func to trigger event by action in compute config -func TriggerEventAction(ctx context.Context, command string, content string) { - handler, exists := actionDispatchMap[command] - if !exists { - logger.Error(ctx, "unknown action command", "command", command) - return - } - err := handler(ctx, content) - if err != nil { - logger.Error(ctx, "action handler failed", "command", command, "content", content, "error", err) - return - } - logger.Info(ctx, "action handler success", "command", command, "content", content) -} - -func handleInfoAction(ctx context.Context, content string) error { - // 实际执行发送警告、记录日志等操作 - actionParams := content - // ... logic to send info level event using actionParams ... - logger.Warn(ctx, "trigger info event", "message", actionParams) - return nil -} - -func handleWarningAction(ctx context.Context, content string) error { - // 实际执行发送警告、记录日志等操作 - actionParams := content - // ... logic to send warning level event using actionParams ... - logger.Warn(ctx, "trigger warning event", "message", actionParams) - return nil -} - -func handleErrorAction(ctx context.Context, content string) error { - // 实际执行发送警告、记录日志等操作 - actionParams := content - // ... logic to send error level event using actionParams ... - logger.Warn(ctx, "trigger error event", "message", actionParams) - return nil -} - -func handleCriticalAction(ctx context.Context, content string) error { - // 实际执行发送警告、记录日志等操作 - actionParams := content - // ... logic to send critical level event using actionParams ... - logger.Warn(ctx, "trigger critical event", "message", actionParams) - return nil -} - -func handleExceptionAction(ctx context.Context, content string) error { - // 实际执行发送警告、记录日志等操作 - actionParams := content - // ... logic to send except level event using actionParams ... - logger.Warn(ctx, "trigger except event", "message", actionParams) - return nil -} diff --git a/real-time-data/real_time_data_computing.go b/real-time-data/real_time_data_computing.go deleted file mode 100644 index 6de1ab1..0000000 --- a/real-time-data/real_time_data_computing.go +++ /dev/null @@ -1,400 +0,0 @@ -// Package realtimedata define real time data operation functions -package realtimedata - -import ( - "context" - "errors" - "fmt" - "time" - - "modelRT/constants" - "modelRT/diagram" - "modelRT/logger" - "modelRT/model" - "modelRT/network" - "modelRT/orm" - "modelRT/util" -) - -var ( - // RealTimeDataChan define channel of real time data receive - RealTimeDataChan chan network.RealTimeDataReceiveRequest - globalComputeState *MeasComputeState -) - -func init() { - RealTimeDataChan = make(chan network.RealTimeDataReceiveRequest, 100) - globalComputeState = NewMeasComputeState() -} - -// StartRealTimeDataComputing define func to start real time data process goroutines by measurement info -func StartRealTimeDataComputing(ctx context.Context, measurements []orm.Measurement) { - for _, measurement := range measurements { - enableValue, exist := measurement.EventPlan["enable"] - enable, ok := enableValue.(bool) - if !exist || !enable { - logger.Info(ctx, "measurement object do not need real time data computing", "measurement_uuid", measurement.ComponentUUID) - continue - } - - if !ok { - logger.Error(ctx, "covert enable variable to boolean type failed", "measurement_uuid", measurement.ComponentUUID, "enable", enableValue) - continue - } - - conf, err := initComputeConfig(measurement) - if err != nil { - logger.Error(ctx, "failed to initialize real time compute config", "measurement_uuid", measurement.ComponentUUID, "error", err) - continue - } - - if conf == nil { - logger.Info(ctx, "measurement object is disabled or does not require real time computing", "measurement_uuid", measurement.ComponentUUID) - continue - } - - uuidStr := measurement.ComponentUUID.String() - enrichedCtx := context.WithValue(ctx, constants.MeasurementUUIDKey, uuidStr) - conf.StopGchan = make(chan struct{}) - globalComputeState.Store(uuidStr, conf) - logger.Info(ctx, "starting real time data computing for measurement", "measurement_uuid", measurement.ComponentUUID) - go continuousComputation(enrichedCtx, conf) - } -} - -func initComputeConfig(measurement orm.Measurement) (*ComputeConfig, error) { - var err error - - enableValue, exist := measurement.EventPlan["enable"] - enable, ok := enableValue.(bool) - if !exist { - return nil, nil - } - - if !ok { - return nil, fmt.Errorf("field enable can not be converted to boolean, found type: %T", enableValue) - } - - if !enable { - return nil, nil - } - - conf := &ComputeConfig{} - - causeValue, exist := measurement.EventPlan["cause"] - if !exist { - return nil, errors.New("missing required field cause") - } - - cause, ok := causeValue.(map[string]any) - if !ok { - return nil, fmt.Errorf("field cause can not be converted to map[string]any, found type: %T", causeValue) - } - conf.Cause, err = processCauseMap(cause) - if err != nil { - return nil, fmt.Errorf("parse content of field cause failed:%w", err) - } - - actionValue, exist := measurement.EventPlan["action"] - if !exist { - return nil, errors.New("missing required field action") - } - action, ok := actionValue.(map[string]any) - if !ok { - return nil, fmt.Errorf("field action can not be converted to map[string]any, found type: %T", actionValue) - } - conf.Action = action - - queryKey, err := model.GenerateMeasureIdentifier(measurement.DataSource) - if err != nil { - return nil, fmt.Errorf("generate redis query key by datasource failed: %w", err) - } - conf.QueryKey = queryKey - conf.DataSize = int64(measurement.Size) - // TODO use constant values ​​for temporary settings - conf.minBreachCount = constants.MinBreachCount - // TODO 后续优化 duration 创建方式 - conf.Duration = 10 - - isFloatCause := false - if _, exists := conf.Cause["up"]; exists { - isFloatCause = true - } else if _, exists := conf.Cause["down"]; exists { - isFloatCause = true - } else if _, exists := conf.Cause["upup"]; exists { - isFloatCause = true - } else if _, exists := conf.Cause["downdown"]; exists { - isFloatCause = true - } - - if isFloatCause { - // te config - teThresholds, err := parseTEThresholds(conf.Cause) - if err != nil { - return nil, fmt.Errorf("failed to parse telemetry thresholds: %w", err) - } - conf.Analyzer = &TEAnalyzer{Thresholds: teThresholds} - } else { - // ti config - tiThresholds, err := parseTIThresholds(conf.Cause) - if err != nil { - return nil, fmt.Errorf("failed to parse telesignal thresholds: %w", err) - } - conf.Analyzer = &TIAnalyzer{Thresholds: tiThresholds} - } - - return conf, nil -} - -func processCauseMap(data map[string]any) (map[string]any, error) { - causeResult := make(map[string]any) - keysToExtract := []string{"up", "down", "upup", "downdown"} - - var foundFloatKey bool - for _, key := range keysToExtract { - if value, exists := data[key]; exists { - - foundFloatKey = true - - // check value type - if floatVal, ok := value.(float64); ok { - causeResult[key] = floatVal - } else { - return nil, fmt.Errorf("key:%s already exists but type is incorrect.expected float64, actual %T", key, value) - } - } - } - - if foundFloatKey == true { - return causeResult, nil - } - - edgeKey := "edge" - if value, exists := data[edgeKey]; exists { - if stringVal, ok := value.(string); ok { - switch stringVal { - case "raising": - fallthrough - case "falling": - causeResult[edgeKey] = stringVal - default: - return nil, fmt.Errorf("key:%s value is incorrect,actual value %s", edgeKey, value) - } - } else { - return nil, fmt.Errorf("key:%s already exists but type is incorrect.expected string, actual %T", edgeKey, value) - } - } else { - return nil, fmt.Errorf("key:%s do not exists", edgeKey) - } - return nil, fmt.Errorf("cause map is invalid: missing required keys (%v) or '%s'", keysToExtract, edgeKey) -} - -func continuousComputation(ctx context.Context, conf *ComputeConfig) { - client := diagram.NewRedisClient() - uuid, _ := ctx.Value(constants.MeasurementUUIDKey).(string) - duration := util.SecondsToDuration(conf.Duration) - ticker := time.NewTicker(duration) - defer ticker.Stop() - - for { - select { - case <-conf.StopGchan: - logger.Info(ctx, "continuous computing groutine stopped by local StopGchan", "uuid", uuid) - return - case <-ctx.Done(): - logger.Info(ctx, "continuous computing goroutine stopped by parent context done signal") - return - case <-ticker.C: - members, err := client.QueryByZRangeByLex(ctx, conf.QueryKey, conf.DataSize) - if err != nil { - logger.Error(ctx, "query real time data from redis failed", "key", conf.QueryKey, "error", err) - continue - } - - realTimedatas := util.ConvertZSetMembersToFloat64(members) - if conf.Analyzer != nil { - conf.Analyzer.AnalyzeAndTriggerEvent(ctx, conf, realTimedatas) - } else { - logger.Error(ctx, "analyzer is not initialized for this measurement", "uuid", uuid) - } - } - } -} - -// // ReceiveChan define func to real time data receive and process -// func ReceiveChan(ctx context.Context, consumerConfig *kafka.ConfigMap, topics []string, duration float32) { -// consumer, err := kafka.NewConsumer(consumerConfig) -// if err != nil { -// logger.Error(ctx, "create kafka consumer failed", "error", err) -// return -// } -// defer consumer.Close() - -// err = consumer.SubscribeTopics(topics, nil) -// if err != nil { -// logger.Error(ctx, "subscribe kafka topics failed", "topic", topics, "error", err) -// return -// } - -// batchSize := 100 -// batchTimeout := util.SecondsToDuration(duration) -// messages := make([]*kafka.Message, 0, batchSize) -// lastCommit := time.Now() -// logger.Info(ctx, "start consuming from kafka", "topic", topics) -// for { -// select { -// case <-ctx.Done(): -// logger.Info(ctx, "stop real time data computing by context cancel") -// return -// case realTimeData := <-RealTimeDataChan: -// componentUUID := realTimeData.PayLoad.ComponentUUID -// component, err := diagram.GetComponentMap(componentUUID) -// if err != nil { -// logger.Error(ctx, "query component info from diagram map by componet id failed", "component_uuid", componentUUID, "error", err) -// continue -// } - -// componentType := component.Type -// if componentType != constants.DemoType { -// logger.Error(ctx, "can not process real time data of component type not equal DemoType", "component_uuid", componentUUID) -// continue -// } - -// var anchorName string -// var compareValUpperLimit, compareValLowerLimit float64 -// var anchorRealTimeData []float64 -// var calculateFunc func(archorValue float64, args ...float64) float64 - -// // calculateFunc, params := config.SelectAnchorCalculateFuncAndParams(componentType, anchorName, componentData) - -// for _, param := range realTimeData.PayLoad.Values { -// anchorRealTimeData = append(anchorRealTimeData, param.Value) -// } - -// anchorConfig := config.AnchorParamConfig{ -// AnchorParamBaseConfig: config.AnchorParamBaseConfig{ -// ComponentUUID: componentUUID, -// AnchorName: anchorName, -// CompareValUpperLimit: compareValUpperLimit, -// CompareValLowerLimit: compareValLowerLimit, -// AnchorRealTimeData: anchorRealTimeData, -// }, -// CalculateFunc: calculateFunc, -// CalculateParams: []float64{}, -// } -// anchorChan, err := pool.GetAnchorParamChan(ctx, componentUUID) -// if err != nil { -// logger.Error(ctx, "get anchor param chan failed", "component_uuid", componentUUID, "error", err) -// continue -// } -// anchorChan <- anchorConfig -// default: -// msg, err := consumer.ReadMessage(batchTimeout) -// if err != nil { -// if err.(kafka.Error).Code() == kafka.ErrTimedOut { -// // process accumulated messages when timeout -// if len(messages) > 0 { -// processMessageBatch(ctx, messages) -// consumer.Commit() -// messages = messages[:0] -// } -// continue -// } -// logger.Error(ctx, "read message from kafka failed", "error", err, "msg", msg) -// continue -// } - -// messages = append(messages, msg) -// // process messages when batch size or timeout period is reached -// if len(messages) >= batchSize || time.Since(lastCommit) >= batchTimeout { -// processMessageBatch(ctx, messages) -// consumer.Commit() -// messages = messages[:0] -// lastCommit = time.Now() -// } -// } -// } -// } - -// type realTimeDataPayload struct { -// ComponentUUID string -// Values []float64 -// } - -// type realTimeData struct { -// Payload realTimeDataPayload -// } - -// func parseKafkaMessage(msgValue []byte) (*realTimeData, error) { -// var realTimeData realTimeData -// err := json.Unmarshal(msgValue, &realTimeData) -// if err != nil { -// return nil, fmt.Errorf("unmarshal real time data failed: %w", err) -// } -// return &realTimeData, nil -// } - -// func processRealTimeData(ctx context.Context, realTimeData *realTimeData) { -// componentUUID := realTimeData.Payload.ComponentUUID -// component, err := diagram.GetComponentMap(componentUUID) -// if err != nil { -// logger.Error(ctx, "query component info from diagram map by component id failed", -// "component_uuid", componentUUID, "error", err) -// return -// } - -// componentType := component.Type -// if componentType != constants.DemoType { -// logger.Error(ctx, "can not process real time data of component type not equal DemoType", -// "component_uuid", componentUUID) -// return -// } - -// var anchorName string -// var compareValUpperLimit, compareValLowerLimit float64 -// var anchorRealTimeData []float64 -// var calculateFunc func(archorValue float64, args ...float64) float64 - -// for _, param := range realTimeData.Payload.Values { -// anchorRealTimeData = append(anchorRealTimeData, param) -// } - -// anchorConfig := config.AnchorParamConfig{ -// AnchorParamBaseConfig: config.AnchorParamBaseConfig{ -// ComponentUUID: componentUUID, -// AnchorName: anchorName, -// CompareValUpperLimit: compareValUpperLimit, -// CompareValLowerLimit: compareValLowerLimit, -// AnchorRealTimeData: anchorRealTimeData, -// }, -// CalculateFunc: calculateFunc, -// CalculateParams: []float64{}, -// } - -// anchorChan, err := pool.GetAnchorParamChan(ctx, componentUUID) -// if err != nil { -// logger.Error(ctx, "get anchor param chan failed", -// "component_uuid", componentUUID, "error", err) -// return -// } - -// select { -// case anchorChan <- anchorConfig: -// case <-ctx.Done(): -// logger.Info(ctx, "context done while sending to anchor chan") -// case <-time.After(5 * time.Second): -// logger.Error(ctx, "timeout sending to anchor chan", "component_uuid", componentUUID) -// } -// } - -// // processMessageBatch define func to bathc process kafka message -// func processMessageBatch(ctx context.Context, messages []*kafka.Message) { -// for _, msg := range messages { -// realTimeData, err := parseKafkaMessage(msg.Value) -// if err != nil { -// logger.Error(ctx, "parse kafka message failed", "error", err, "msg", msg) -// continue -// } -// go processRealTimeData(ctx, realTimeData) -// } -// } diff --git a/real-time-data/real_time_data_up_down_limit_computing.go b/real-time-data/real_time_data_up_down_limit_computing.go new file mode 100644 index 0000000..24a93e9 --- /dev/null +++ b/real-time-data/real_time_data_up_down_limit_computing.go @@ -0,0 +1,249 @@ +// Package realtimedata define real time data operation functions +package realtimedata + +import ( + "context" + "errors" + "fmt" + "time" + + "modelRT/constants" + "modelRT/diagram" + "modelRT/logger" + "modelRT/model" + "modelRT/network" + "modelRT/orm" + "modelRT/util" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + oteltrace "go.opentelemetry.io/otel/trace" +) + +var ( + // RealTimeDataChan define channel of real time data receive + RealTimeDataChan chan network.RealTimeDataReceiveRequest + globalComputeState *MeasComputeState +) + +func init() { + RealTimeDataChan = make(chan network.RealTimeDataReceiveRequest, 100) + globalComputeState = NewMeasComputeState() +} + +// StartComputingRealTimeDataLimit define func to start compute real time data up or down limit process goroutines by measurement info +func StartComputingRealTimeDataLimit(ctx context.Context, measurements []orm.Measurement) { + for _, measurement := range measurements { + enableValue, exist := measurement.EventPlan["enable"] + enable, ok := enableValue.(bool) + if !exist || !enable { + logger.Info(ctx, "measurement object do not need real time data computing", "measurement_uuid", measurement.ComponentUUID) + continue + } + + if !ok { + logger.Error(ctx, "covert enable variable to boolean type failed", "measurement_uuid", measurement.ComponentUUID, "enable", enableValue) + continue + } + + conf, err := initComputeConfig(measurement) + if err != nil { + logger.Error(ctx, "failed to initialize real time compute config", "measurement_uuid", measurement.ComponentUUID, "error", err) + continue + } + + if conf == nil { + logger.Info(ctx, "measurement object is disabled or does not require real time computing", "measurement_uuid", measurement.ComponentUUID) + continue + } + + uuidStr := measurement.ComponentUUID.String() + enrichedCtx := context.WithValue(ctx, constants.MeasurementUUIDKey, uuidStr) + conf.StopGchan = make(chan struct{}) + globalComputeState.Store(uuidStr, conf) + logger.Info(ctx, "starting computing real time data limit for measurement", "measurement_uuid", measurement.ComponentUUID) + go continuousComputation(enrichedCtx, conf) + } +} + +func initComputeConfig(measurement orm.Measurement) (*ComputeConfig, error) { + var err error + + enableValue, exist := measurement.EventPlan["enable"] + enable, ok := enableValue.(bool) + if !exist { + return nil, nil + } + + if !ok { + return nil, fmt.Errorf("field enable can not be converted to boolean, found type: %T", enableValue) + } + + if !enable { + return nil, nil + } + + conf := &ComputeConfig{} + + causeValue, exist := measurement.EventPlan["cause"] + if !exist { + return nil, errors.New("missing required field cause") + } + + cause, ok := causeValue.(map[string]any) + if !ok { + return nil, fmt.Errorf("field cause can not be converted to map[string]any, found type: %T", causeValue) + } + conf.Cause, err = processCauseMap(cause) + if err != nil { + return nil, fmt.Errorf("parse content of field cause failed:%w", err) + } + + actionValue, exist := measurement.EventPlan["action"] + if !exist { + return nil, errors.New("missing required field action") + } + action, ok := actionValue.(map[string]any) + if !ok { + return nil, fmt.Errorf("field action can not be converted to map[string]any, found type: %T", actionValue) + } + conf.Action = action + + queryKey, err := model.GenerateMeasureIdentifier(measurement.DataSource) + if err != nil { + return nil, fmt.Errorf("generate redis query key by datasource failed: %w", err) + } + conf.QueryKey = queryKey + conf.DataSize = int64(measurement.Size) + // TODO use constant values ​​for temporary settings + conf.minBreachCount = constants.MinBreachCount + // TODO 后续优化 duration 创建方式 + conf.Duration = 10 + + isFloatCause := false + if _, exists := conf.Cause["up"]; exists { + isFloatCause = true + } else if _, exists := conf.Cause["down"]; exists { + isFloatCause = true + } else if _, exists := conf.Cause["upup"]; exists { + isFloatCause = true + } else if _, exists := conf.Cause["downdown"]; exists { + isFloatCause = true + } + + if isFloatCause { + // te config + teThresholds, err := parseTEThresholds(conf.Cause) + if err != nil { + return nil, fmt.Errorf("failed to parse telemetry thresholds: %w", err) + } + conf.Analyzer = &TEAnalyzer{Thresholds: teThresholds} + } else { + // ti config + tiThresholds, err := parseTIThresholds(conf.Cause) + if err != nil { + return nil, fmt.Errorf("failed to parse telesignal thresholds: %w", err) + } + conf.Analyzer = &TIAnalyzer{Thresholds: tiThresholds} + } + + return conf, nil +} + +func processCauseMap(data map[string]any) (map[string]any, error) { + causeResult := make(map[string]any) + keysToExtract := []string{"up", "down", "upup", "downdown"} + + var foundFloatKey bool + for _, key := range keysToExtract { + if value, exists := data[key]; exists { + + foundFloatKey = true + + // check value type + if floatVal, ok := value.(float64); ok { + causeResult[key] = floatVal + } else { + return nil, fmt.Errorf("key:%s already exists but type is incorrect.expected float64, actual %T", key, value) + } + } + } + + if foundFloatKey { + return causeResult, nil + } + + edgeKey := "edge" + if value, exists := data[edgeKey]; exists { + if stringVal, ok := value.(string); ok { + switch stringVal { + case "raising": + fallthrough + case "falling": + causeResult[edgeKey] = stringVal + default: + return nil, fmt.Errorf("key:%s value is incorrect,actual value %s", edgeKey, value) + } + } else { + return nil, fmt.Errorf("key:%s already exists but type is incorrect.expected string, actual %T", edgeKey, value) + } + } else { + return nil, fmt.Errorf("key:%s do not exists", edgeKey) + } + return nil, fmt.Errorf("cause map is invalid: missing required keys (%v) or '%s'", keysToExtract, edgeKey) +} + +func continuousComputation(ctx context.Context, conf *ComputeConfig) { + client := diagram.NewRedisClient() + uuid, _ := ctx.Value(constants.MeasurementUUIDKey).(string) + duration := util.SecondsToDuration(conf.Duration) + ticker := time.NewTicker(duration) + defer ticker.Stop() + + for { + select { + case <-conf.StopGchan: + logger.Info(ctx, "continuous computing groutine stopped by local StopGchan", "uuid", uuid) + return + case <-ctx.Done(): + logger.Info(ctx, "continuous computing goroutine stopped by parent context done signal") + return + case <-ticker.C: + // start a new root span for this computation cycle, linked to the startup span + startupSpanCtx := oteltrace.SpanFromContext(ctx).SpanContext() + cycleCtx, cycleSpan := otel.Tracer("modelRT/realtime").Start( + context.Background(), + "realtime.compute_cycle", + oteltrace.WithNewRoot(), + oteltrace.WithLinks(oteltrace.Link{SpanContext: startupSpanCtx}), + oteltrace.WithAttributes( + attribute.String("measurement_uuid", uuid), + attribute.String("query_key", conf.QueryKey), + ), + ) + + queryCtx, cancel := context.WithTimeout(cycleCtx, 2*time.Second) + members, err := client.QueryByZRange(queryCtx, conf.QueryKey, conf.DataSize) + cancel() + if err != nil { + logger.Error(cycleCtx, "query real time data from redis failed", "key", conf.QueryKey, "error", err) + cycleSpan.End() + continue + } + + realTimedatas := util.ConvertZSetMembersToFloat64(members) + if len(realTimedatas) == 0 { + logger.Info(cycleCtx, "no real time data queried from redis, skip this computation cycle", "key", conf.QueryKey) + cycleSpan.End() + continue + } + + if conf.Analyzer != nil { + conf.Analyzer.AnalyzeAndTriggerEvent(cycleCtx, conf, realTimedatas) + } else { + logger.Error(cycleCtx, "analyzer is not initialized for this measurement", "uuid", uuid) + } + cycleSpan.End() + } + } +} diff --git a/router/async_task.go b/router/async_task.go new file mode 100644 index 0000000..4b0861c --- /dev/null +++ b/router/async_task.go @@ -0,0 +1,32 @@ +// Package router provides router config +package router + +import ( + "modelRT/handler" + + "github.com/gin-gonic/gin" +) + +// registerAsyncTaskRoutes define func of register async task routes +func registerAsyncTaskRoutes(rg *gin.RouterGroup, middlewares ...gin.HandlerFunc) { + g := rg.Group("/task/") + g.Use(middlewares...) + + // Async task creation + g.POST("async", handler.AsyncTaskCreateHandler) + + // Async task result query + g.GET("async/results", handler.AsyncTaskResultQueryHandler) + + // Async task detail query + g.GET("async/:task_id", handler.AsyncTaskResultDetailHandler) + + // Async task cancellation + g.POST("async/:task_id/cancel", handler.AsyncTaskCancelHandler) + + // Internal APIs for worker updates (not exposed to external users) + internal := g.Group("internal/") + internal.Use(middlewares...) + internal.POST("async/progress", handler.AsyncTaskProgressUpdateHandler) + internal.POST("async/status", handler.AsyncTaskStatusUpdateHandler) +} \ No newline at end of file diff --git a/router/router.go b/router/router.go index 6fbc113..f242cb9 100644 --- a/router/router.go +++ b/router/router.go @@ -27,4 +27,5 @@ func RegisterRoutes(engine *gin.Engine, clientToken string) { registerDataRoutes(routeGroup) registerMonitorRoutes(routeGroup) registerComponentRoutes(routeGroup, middleware.SetTokenMiddleware(clientToken)) + registerAsyncTaskRoutes(routeGroup, middleware.SetTokenMiddleware(clientToken)) } diff --git a/sharememory/share_memeory.go b/sharememory/share_memeory.go deleted file mode 100644 index 8248064..0000000 --- a/sharememory/share_memeory.go +++ /dev/null @@ -1,97 +0,0 @@ -package sharememory - -import ( - "fmt" - "unsafe" - - "modelRT/orm" - - "golang.org/x/sys/unix" -) - -// CreateShareMemory defines a function to create a shared memory -func CreateShareMemory(key uintptr, structSize uintptr) (uintptr, error) { - // logger := logger.GetLoggerInstance() - // create shared memory - shmID, _, err := unix.Syscall(unix.SYS_SHMGET, key, structSize, unix.IPC_CREAT|0o666) - if err != 0 { - // logger.Error(fmt.Sprintf("create shared memory by key %v failed:", key), zap.Error(err)) - return 0, fmt.Errorf("create shared memory failed:%w", err) - } - - // attach shared memory - shmAddr, _, err := unix.Syscall(unix.SYS_SHMAT, shmID, 0, 0) - if err != 0 { - // logger.Error(fmt.Sprintf("attach shared memory by shmID %v failed:", shmID), zap.Error(err)) - return 0, fmt.Errorf("attach shared memory failed:%w", err) - } - return shmAddr, nil -} - -// ReadComponentFromShareMemory defines a function to read component value from shared memory -func ReadComponentFromShareMemory(key uintptr, componentInfo *orm.Component) error { - structSize := unsafe.Sizeof(orm.Component{}) - shmID, _, err := unix.Syscall(unix.SYS_SHMGET, key, uintptr(int(structSize)), 0o666) - if err != 0 { - return fmt.Errorf("get shared memory failed:%w", err) - } - - shmAddr, _, err := unix.Syscall(unix.SYS_SHMAT, shmID, 0, 0) - if err != 0 { - return fmt.Errorf("attach shared memory failed:%w", err) - } - - // 读取共享内存中的数据 - componentInfo = (*orm.Component)(unsafe.Pointer(shmAddr + structSize)) - - // Detach shared memory - unix.Syscall(unix.SYS_SHMDT, shmAddr, 0, 0) - return nil -} - -func WriteComponentInShareMemory(key uintptr, componentInfo *orm.Component) error { - structSize := unsafe.Sizeof(orm.Component{}) - shmID, _, err := unix.Syscall(unix.SYS_SHMGET, key, uintptr(int(structSize)), 0o666) - if err != 0 { - return fmt.Errorf("get shared memory failed:%w", err) - } - - shmAddr, _, err := unix.Syscall(unix.SYS_SHMAT, shmID, 0, 0) - if err != 0 { - return fmt.Errorf("attach shared memory failed:%w", err) - } - - obj := (*orm.Component)(unsafe.Pointer(shmAddr + unsafe.Sizeof(structSize))) - fmt.Println(obj) - - // id integer NOT NULL DEFAULT nextval('component_id_seq'::regclass), - // global_uuid uuid NOT NULL DEFAULT gen_random_uuid(), - // nspath character varying(32) COLLATE pg_catalog."default", - // tag character varying(32) COLLATE pg_catalog."default" NOT NULL, - // name character varying(64) COLLATE pg_catalog."default" NOT NULL, - // description character varying(512) COLLATE pg_catalog."default" NOT NULL DEFAULT ''::character varying, - // grid character varying(64) COLLATE pg_catalog."default" NOT NULL, - // zone character varying(64) COLLATE pg_catalog."default" NOT NULL, - // station character varying(64) COLLATE pg_catalog."default" NOT NULL, - // type integer NOT NULL, - // in_service boolean DEFAULT false, - // state integer NOT NULL DEFAULT 0, - // connected_bus jsonb NOT NULL DEFAULT '{}'::jsonb, - // label jsonb NOT NULL DEFAULT '{}'::jsonb, - // context jsonb NOT NULL DEFAULT '{}'::jsonb, - // page_id integer NOT NULL, - // op integer NOT NULL DEFAULT '-1'::integer, - // ts timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, - - unix.Syscall(unix.SYS_SHMDT, shmAddr, 0, 0) - return nil -} - -// DeleteShareMemory defines a function to delete shared memory -func DeleteShareMemory(key uintptr) error { - _, _, err := unix.Syscall(unix.SYS_SHM_UNLINK, key, 0, 0o666) - if err != 0 { - return fmt.Errorf("get shared memory failed:%w", err) - } - return nil -} diff --git a/sql/async_task.sql b/sql/async_task.sql new file mode 100644 index 0000000..8b2acf9 --- /dev/null +++ b/sql/async_task.sql @@ -0,0 +1,57 @@ +-- Async task table schema migration +-- Add new columns for enhanced task tracking and retry functionality + +-- Add new columns to async_task table +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS started_at bigint NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS execution_time bigint NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS retry_count integer NOT NULL DEFAULT 0; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS max_retry_count integer NOT NULL DEFAULT 3; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS next_retry_time bigint NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS retry_delay integer NOT NULL DEFAULT 5000; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS priority integer NOT NULL DEFAULT 5; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS queue_name varchar(100) NOT NULL DEFAULT 'default'; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS worker_id varchar(50) NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS failure_reason text NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS stack_trace text NULL; +ALTER TABLE async_task ADD COLUMN IF NOT EXISTS created_by varchar(100) NULL; + +-- Add new columns to async_task_result table +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS execution_time bigint NOT NULL DEFAULT 0; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS memory_usage bigint NULL; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS cpu_usage double precision NULL; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS retry_count integer NOT NULL DEFAULT 0; +ALTER TABLE async_task_result ADD COLUMN IF NOT EXISTS completed_at bigint NOT NULL DEFAULT 0; + +-- Add indexes for improved query performance +CREATE INDEX IF NOT EXISTS idx_async_task_status_priority ON async_task(status, priority DESC); +CREATE INDEX IF NOT EXISTS idx_async_task_next_retry_time ON async_task(next_retry_time) WHERE status = 'FAILED'; +CREATE INDEX IF NOT EXISTS idx_async_task_created_by ON async_task(created_by); +CREATE INDEX IF NOT EXISTS idx_async_task_task_type ON async_task(task_type); +CREATE INDEX IF NOT EXISTS idx_async_task_started_at ON async_task(started_at) WHERE started_at IS NOT NULL; + +-- Update existing rows to have default values for new columns +UPDATE async_task SET priority = 5 WHERE priority IS NULL; +UPDATE async_task SET queue_name = 'default' WHERE queue_name IS NULL; +UPDATE async_task SET retry_count = 0 WHERE retry_count IS NULL; +UPDATE async_task SET max_retry_count = 3 WHERE max_retry_count IS NULL; +UPDATE async_task SET retry_delay = 5000 WHERE retry_delay IS NULL; + +-- Add comments for new columns +COMMENT ON COLUMN async_task.started_at IS 'Timestamp when task execution started (Unix epoch seconds)'; +COMMENT ON COLUMN async_task.execution_time IS 'Task execution time in milliseconds'; +COMMENT ON COLUMN async_task.retry_count IS 'Number of retry attempts for failed tasks'; +COMMENT ON COLUMN async_task.max_retry_count IS 'Maximum number of retry attempts allowed'; +COMMENT ON COLUMN async_task.next_retry_time IS 'Next retry timestamp (Unix epoch seconds)'; +COMMENT ON COLUMN async_task.retry_delay IS 'Delay between retries in milliseconds'; +COMMENT ON COLUMN async_task.priority IS 'Task priority (1-10, higher is more important)'; +COMMENT ON COLUMN async_task.queue_name IS 'Name of the queue the task belongs to'; +COMMENT ON COLUMN async_task.worker_id IS 'ID of the worker processing the task'; +COMMENT ON COLUMN async_task.failure_reason IS 'Reason for task failure'; +COMMENT ON COLUMN async_task.stack_trace IS 'Stack trace for debugging failed tasks'; +COMMENT ON COLUMN async_task.created_by IS 'User or system that created the task'; + +COMMENT ON COLUMN async_task_result.execution_time IS 'Total execution time in milliseconds'; +COMMENT ON COLUMN async_task_result.memory_usage IS 'Memory usage in bytes'; +COMMENT ON COLUMN async_task_result.cpu_usage IS 'CPU usage percentage'; +COMMENT ON COLUMN async_task_result.retry_count IS 'Number of retries before success'; +COMMENT ON COLUMN async_task_result.completed_at IS 'Timestamp when task completed (Unix epoch seconds)'; \ No newline at end of file diff --git a/task/base_task.go b/task/base_task.go new file mode 100644 index 0000000..9d29a1a --- /dev/null +++ b/task/base_task.go @@ -0,0 +1,61 @@ +// Package task provides unified task type definitions and interfaces +package task + +import ( + "context" + "fmt" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// Params defines the interface for task-specific parameters +type Params interface { + Validate() error + GetType() UnifiedTaskType + ToMap() map[string]any + FromMap(params map[string]any) error +} + +// BaseTask provides common functionality for all task implementations +type BaseTask struct { + taskType UnifiedTaskType + params Params + name string +} + +// NewBaseTask creates a new BaseTask instance +func NewBaseTask(taskType UnifiedTaskType, params Params, name string) *BaseTask { + return &BaseTask{ + taskType: taskType, + params: params, + name: name, + } +} + +func (t *BaseTask) GetType() UnifiedTaskType { + return t.taskType +} + +func (t *BaseTask) GetParams() Params { + return t.params +} + +func (t *BaseTask) GetName() string { + return t.name +} + +func (t *BaseTask) Validate() error { + if t.params == nil { + return fmt.Errorf("task parameters cannot be nil") + } + if t.taskType != t.params.GetType() { + return fmt.Errorf("task type mismatch: expected %s, got %s", t.taskType, t.params.GetType()) + } + return t.params.Validate() +} + +// Execute is a placeholder; concrete task types override this via embedding. +func (t *BaseTask) Execute(_ context.Context, _ uuid.UUID, _ *gorm.DB) error { + return fmt.Errorf("Execute not implemented for task type %s", t.taskType) +} diff --git a/task/handler_factory.go b/task/handler_factory.go new file mode 100644 index 0000000..b705e27 --- /dev/null +++ b/task/handler_factory.go @@ -0,0 +1,396 @@ +// Package task provides asynchronous task processing with handler factory pattern +package task + +import ( + "context" + "fmt" + "sync" + "time" + + "modelRT/database" + "modelRT/logger" + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// AsyncTaskHandler defines the interface for task processors +type AsyncTaskHandler interface { + // Execute processes a task with the given ID and params from the MQ message + Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error + // CanHandle returns true if this handler can process the given task type + CanHandle(taskType TaskType) bool + // Name returns the name of the handler for logging and metrics + Name() string +} + +// HandlerFactory creates task handlers based on task type +type HandlerFactory struct { + handlers map[TaskType]AsyncTaskHandler + mu sync.RWMutex +} + +// NewHandlerFactory creates a new HandlerFactory +func NewHandlerFactory() *HandlerFactory { + return &HandlerFactory{ + handlers: make(map[TaskType]AsyncTaskHandler), + } +} + +// RegisterHandler registers a handler for a specific task type +func (f *HandlerFactory) RegisterHandler(ctx context.Context, taskType TaskType, handler AsyncTaskHandler) { + f.mu.Lock() + defer f.mu.Unlock() + + f.handlers[taskType] = handler + logger.Info(ctx, "Handler registered", + "task_type", taskType, + "handler_name", handler.Name(), + ) +} + +// GetHandler returns a handler for the given task type +func (f *HandlerFactory) GetHandler(taskType TaskType) (AsyncTaskHandler, error) { + f.mu.RLock() + handler, exists := f.handlers[taskType] + f.mu.RUnlock() + + if !exists { + return nil, fmt.Errorf("no handler registered for task type: %s", taskType) + } + + return handler, nil +} + +// CreateDefaultHandlers registers all default task handlers +func (f *HandlerFactory) CreateDefaultHandlers(ctx context.Context) { + f.RegisterHandler(ctx, TypeTopologyAnalysis, NewTopologyAnalysisHandler()) + f.RegisterHandler(ctx, TypeEventAnalysis, NewEventAnalysisHandler()) + f.RegisterHandler(ctx, TypeBatchImport, NewBatchImportHandler()) + f.RegisterHandler(ctx, TypeTest, NewTestTaskHandler()) +} + +// BaseHandler provides common functionality for all task handlers +type BaseHandler struct { + name string +} + +// NewBaseHandler creates a new BaseHandler +func NewBaseHandler(name string) *BaseHandler { + return &BaseHandler{name: name} +} + +// Name returns the handler name +func (h *BaseHandler) Name() string { + return h.name +} + +// TopologyAnalysisHandler handles topology analysis tasks +type TopologyAnalysisHandler struct { + BaseHandler +} + +// NewTopologyAnalysisHandler creates a new TopologyAnalysisHandler +func NewTopologyAnalysisHandler() *TopologyAnalysisHandler { + return &TopologyAnalysisHandler{ + BaseHandler: *NewBaseHandler("topology_analysis_handler"), + } +} + +// Execute processes a topology analysis task. +// Params (all sourced from the MQ message, no DB lookup needed): +// - start_component_uuid (string, required): BFS origin +// - end_component_uuid (string, required): reachability target +// - check_in_service (bool, optional, default true): skip out-of-service components +func (h *TopologyAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { + logger.Info(ctx, "topology analysis started", "task_id", taskID) + + // Phase 1: parse params from MQ message + startComponentUUID, endComponentUUID, checkInService, err := parseTopologyAnalysisParams(params) + if err != nil { + return fmt.Errorf("invalid topology analysis params: %w", err) + } + + logger.Info(ctx, "topology params parsed", + "task_id", taskID, + "start", startComponentUUID, + "end", endComponentUUID, + "check_in_service", checkInService, + ) + + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 20); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 20, "error", err) + } + + // Phase 2: query topology edges from startComponentUUID, build adjacency list + topoEdges, err := database.QueryTopologicByStartUUID(ctx, db, startComponentUUID) + if err != nil { + return fmt.Errorf("query topology from start node: %w", err) + } + + // adjacency list: uuid_from → []uuid_to + adjMap := make(map[uuid.UUID][]uuid.UUID, len(topoEdges)) + // collect all UUIDs for batch InService query + allUUIDs := make(map[uuid.UUID]struct{}, len(topoEdges)*2) + allUUIDs[startComponentUUID] = struct{}{} + for _, edge := range topoEdges { + adjMap[edge.UUIDFrom] = append(adjMap[edge.UUIDFrom], edge.UUIDTo) + allUUIDs[edge.UUIDFrom] = struct{}{} + allUUIDs[edge.UUIDTo] = struct{}{} + } + + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 40); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 40, "error", err) + } + + // Phase 3: batch-load InService status (only when checkInService is true) + inServiceMap := make(map[uuid.UUID]bool) + if checkInService { + uuidSlice := make([]uuid.UUID, 0, len(allUUIDs)) + for id := range allUUIDs { + uuidSlice = append(uuidSlice, id) + } + inServiceMap, err = database.QueryComponentsInServiceByUUIDs(ctx, db, uuidSlice) + if err != nil { + return fmt.Errorf("query component in_service status: %w", err) + } + + // check the start node itself before BFS + if !inServiceMap[startComponentUUID] { + return persistTopologyResult(ctx, db, taskID, startComponentUUID, endComponentUUID, + checkInService, false, nil, &startComponentUUID) + } + } + + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 60); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 60, "error", err) + } + + // Phase 4: BFS reachability check + visited := make(map[uuid.UUID]struct{}) + parent := make(map[uuid.UUID]uuid.UUID) // for path reconstruction + queue := []uuid.UUID{startComponentUUID} + visited[startComponentUUID] = struct{}{} + isReachable := false + var blockedBy *uuid.UUID + + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + + if cur == endComponentUUID { + isReachable = true + break + } + + for _, next := range adjMap[cur] { + if _, seen := visited[next]; seen { + continue + } + if checkInService && !inServiceMap[next] { + // record first out-of-service blocker but keep searching other branches + if blockedBy == nil { + id := next + blockedBy = &id + } + continue + } + visited[next] = struct{}{} + parent[next] = cur + queue = append(queue, next) + } + } + + if err := database.UpdateAsyncTaskProgress(ctx, db, taskID, 80); err != nil { + logger.Warn(ctx, "update progress failed", "task_id", taskID, "progress", 80, "error", err) + } + + // Phase 5: reconstruct path (if reachable) and persist result + var path []uuid.UUID + if isReachable { + blockedBy = nil // reachable path found — clear any partial blocker + path = reconstructPath(parent, startComponentUUID, endComponentUUID) + } + + return persistTopologyResult(ctx, db, taskID, startComponentUUID, endComponentUUID, + checkInService, isReachable, path, blockedBy) +} + +// parseTopologyAnalysisParams extracts and validates the three required fields. +// check_in_service defaults to true when absent. +func parseTopologyAnalysisParams(params map[string]any) (startID, endID uuid.UUID, checkInService bool, err error) { + startStr, ok := params["start_component_uuid"].(string) + if !ok || startStr == "" { + err = fmt.Errorf("missing or invalid start_component_uuid") + return + } + endStr, ok := params["end_component_uuid"].(string) + if !ok || endStr == "" { + err = fmt.Errorf("missing or invalid end_component_uuid") + return + } + startID, err = uuid.FromString(startStr) + if err != nil { + err = fmt.Errorf("parse start_component_uuid %q: %w", startStr, err) + return + } + endID, err = uuid.FromString(endStr) + if err != nil { + err = fmt.Errorf("parse end_component_uuid %q: %w", endStr, err) + return + } + + // check_in_service defaults to true + checkInService = true + if v, exists := params["check_in_service"]; exists { + if b, isBool := v.(bool); isBool { + checkInService = b + } + } + return +} + +// reconstructPath walks the parent map backwards from end to start. +func reconstructPath(parent map[uuid.UUID]uuid.UUID, start, end uuid.UUID) []uuid.UUID { + var path []uuid.UUID + for cur := end; cur != start; cur = parent[cur] { + path = append(path, cur) + } + path = append(path, start) + // reverse: path was built end→start + for i, j := 0, len(path)-1; i < j; i, j = i+1, j-1 { + path[i], path[j] = path[j], path[i] + } + return path +} + +// persistTopologyResult serialises the analysis outcome and writes it to async_task_result. +func persistTopologyResult( + ctx context.Context, db *gorm.DB, taskID uuid.UUID, + startID, endID uuid.UUID, checkInService, isReachable bool, + path []uuid.UUID, blockedBy *uuid.UUID, +) error { + pathStrs := make([]string, 0, len(path)) + for _, id := range path { + pathStrs = append(pathStrs, id.String()) + } + + result := orm.JSONMap{ + "start_component_uuid": startID.String(), + "end_component_uuid": endID.String(), + "check_in_service": checkInService, + "is_reachable": isReachable, + "path": pathStrs, + "computed_at": time.Now().Unix(), + } + if blockedBy != nil { + result["blocked_by"] = blockedBy.String() + } + + if err := database.CreateAsyncTaskResult(ctx, db, taskID, result); err != nil { + return fmt.Errorf("save task result: %w", err) + } + + logger.Info(ctx, "topology analysis completed", + "task_id", taskID, + "is_reachable", isReachable, + "path_length", len(path), + ) + return nil +} + +// CanHandle returns true for topology analysis tasks +func (h *TopologyAnalysisHandler) CanHandle(taskType TaskType) bool { + return taskType == TypeTopologyAnalysis +} + +// EventAnalysisHandler handles event analysis tasks +type EventAnalysisHandler struct { + BaseHandler +} + +// NewEventAnalysisHandler creates a new EventAnalysisHandler +func NewEventAnalysisHandler() *EventAnalysisHandler { + return &EventAnalysisHandler{ + BaseHandler: *NewBaseHandler("event_analysis_handler"), + } +} + +// Execute processes an event analysis task +func (h *EventAnalysisHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { + logger.Info(ctx, "Starting event analysis", + "task_id", taskID, + "task_params", params, + ) + + // TODO: Implement actual event analysis logic + // This would typically involve: + // 1. Fetching motor and trigger information + // 2. Analyzing events within the specified duration + // 3. Generating analysis report + // 4. Storing results in database + + // Simulate work + logger.Info(ctx, "Event analysis completed", + "task_id", taskID, + "task_params", params, + "db", db, + ) + + return nil +} + +// CanHandle returns true for event analysis tasks +func (h *EventAnalysisHandler) CanHandle(taskType TaskType) bool { + return taskType == TypeEventAnalysis +} + +// BatchImportHandler handles batch import tasks +type BatchImportHandler struct { + BaseHandler +} + +// NewBatchImportHandler creates a new BatchImportHandler +func NewBatchImportHandler() *BatchImportHandler { + return &BatchImportHandler{ + BaseHandler: *NewBaseHandler("batch_import_handler"), + } +} + +// Execute processes a batch import task +func (h *BatchImportHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { + logger.Info(ctx, "Starting batch import", + "task_id", taskID, + "task_params", params, + "db", db, + ) + + // TODO: Implement actual batch import logic + // This would typically involve: + // 1. Reading file from specified path + // 2. Parsing file content (CSV, Excel, etc.) + // 3. Validating and importing data into database + // 4. Generating import report + + // Simulate work + logger.Info(ctx, "Batch import completed", + "task_id", taskID, + "task_params", params, + "db", db, + ) + + return nil +} + +// CanHandle returns true for batch import tasks +func (h *BatchImportHandler) CanHandle(taskType TaskType) bool { + return taskType == TypeBatchImport +} + +// DefaultHandlerFactory returns a HandlerFactory with all default handlers registered +func DefaultHandlerFactory(ctx context.Context) *HandlerFactory { + factory := NewHandlerFactory() + factory.CreateDefaultHandlers(ctx) + return factory +} diff --git a/task/initializer.go b/task/initializer.go new file mode 100644 index 0000000..008d721 --- /dev/null +++ b/task/initializer.go @@ -0,0 +1,39 @@ +// Package task provides asynchronous task processing with worker pools +package task + +import ( + "context" + "fmt" + + "modelRT/config" + "modelRT/logger" + "gorm.io/gorm" +) + +// InitTaskWorker initializes a task worker with the given configuration and database connection +func InitTaskWorker(ctx context.Context, config config.ModelRTConfig, db *gorm.DB) (*TaskWorker, error) { + // Create worker configuration + workerCfg := WorkerConfig{ + PoolSize: config.AsyncTaskConfig.WorkerPoolSize, + PreAlloc: true, + MaxBlockingTasks: 100, + QueueConsumerCount: config.AsyncTaskConfig.QueueConsumerCount, + PollingInterval: config.AsyncTaskConfig.HealthCheckInterval, + } + + // Create task handler factory + handlerFactory := DefaultHandlerFactory(ctx) + + // Create task worker + worker, err := NewTaskWorker(ctx, workerCfg, db, config.RabbitMQConfig, handlerFactory) + if err != nil { + return nil, fmt.Errorf("failed to create task worker: %w", err) + } + + logger.Info(ctx, "Task worker initialized", + "worker_pool_size", workerCfg.PoolSize, + "queue_consumers", workerCfg.QueueConsumerCount, + ) + + return worker, nil +} diff --git a/task/metrics_logger.go b/task/metrics_logger.go new file mode 100644 index 0000000..d8c0675 --- /dev/null +++ b/task/metrics_logger.go @@ -0,0 +1,157 @@ +// Package task provides metrics logging for asynchronous task system +package task + +import ( + "context" + "runtime" + "time" + + "modelRT/logger" +) + +// MetricsLogger logs task system metrics using the existing logging system +type MetricsLogger struct { + ctx context.Context +} + +// NewMetricsLogger creates a new MetricsLogger +func NewMetricsLogger(ctx context.Context) *MetricsLogger { + return &MetricsLogger{ctx: ctx} +} + +// LogTaskMetrics records task processing metrics +func (m *MetricsLogger) LogTaskMetrics(taskType TaskType, status string, processingTime time.Duration, retryCount int) { + logger.Info(m.ctx, "Task metrics", + "task_type", taskType, + "status", status, + "processing_time_ms", processingTime.Milliseconds(), + "retry_count", retryCount, + "metric_type", "task_processing", + "timestamp", time.Now().Unix(), + ) +} + +// LogQueueMetrics records queue metrics +func (m *MetricsLogger) LogQueueMetrics(queueDepth int, queueLatency time.Duration) { + logger.Info(m.ctx, "Queue metrics", + "queue_depth", queueDepth, + "queue_latency_ms", queueLatency.Milliseconds(), + "metric_type", "queue", + "timestamp", time.Now().Unix(), + ) +} + +// LogWorkerMetrics records worker metrics +func (m *MetricsLogger) LogWorkerMetrics(activeWorkers, idleWorkers, totalWorkers int, memoryUsage uint64, cpuLoad float64) { + logger.Info(m.ctx, "Worker metrics", + "active_workers", activeWorkers, + "idle_workers", idleWorkers, + "total_workers", totalWorkers, + "memory_usage_mb", memoryUsage/(1024*1024), + "cpu_load_percent", cpuLoad, + "metric_type", "worker", + "timestamp", time.Now().Unix(), + ) +} + +// LogRetryMetrics records retry metrics +func (m *MetricsLogger) LogRetryMetrics(taskType TaskType, retryCount int, success bool, delay time.Duration) { + logger.Info(m.ctx, "Retry metrics", + "task_type", taskType, + "retry_count", retryCount, + "retry_success", success, + "retry_delay_ms", delay.Milliseconds(), + "metric_type", "retry", + "timestamp", time.Now().Unix(), + ) +} + +// LogSystemMetrics records system-level metrics (memory, CPU, goroutines) +func (m *MetricsLogger) LogSystemMetrics() { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + logger.Info(m.ctx, "System metrics", + "metric_type", "system", + "timestamp", time.Now().Unix(), + "goroutines", runtime.NumGoroutine(), + "memory_alloc_mb", memStats.Alloc/(1024*1024), + "memory_total_alloc_mb", memStats.TotalAlloc/(1024*1024), + "memory_sys_mb", memStats.Sys/(1024*1024), + "memory_heap_alloc_mb", memStats.HeapAlloc/(1024*1024), + "memory_heap_sys_mb", memStats.HeapSys/(1024*1024), + "memory_heap_inuse_mb", memStats.HeapInuse/(1024*1024), + "gc_pause_total_ns", memStats.PauseTotalNs, + "num_gc", memStats.NumGC, + ) +} + +// LogTaskCompletionMetrics records detailed task completion metrics +func (m *MetricsLogger) LogTaskCompletionMetrics(taskID, taskType, status string, startTime, endTime time.Time, retryCount int, errorMsg string) { + duration := endTime.Sub(startTime) + + logger.Info(m.ctx, "Task completion metrics", + "metric_type", "task_completion", + "timestamp", time.Now().Unix(), + "task_id", taskID, + "task_type", taskType, + "status", status, + "duration_ms", duration.Milliseconds(), + "start_time", startTime.Unix(), + "end_time", endTime.Unix(), + "retry_count", retryCount, + "has_error", errorMsg != "", + "error_msg", errorMsg, + ) +} + +// LogHealthCheckMetrics records health check metrics +func (m *MetricsLogger) LogHealthCheckMetrics(healthy bool, checkDuration time.Duration, components map[string]bool) { + logger.Info(m.ctx, "Health check metrics", + "metric_type", "health_check", + "timestamp", time.Now().Unix(), + "healthy", healthy, + "check_duration_ms", checkDuration.Milliseconds(), + "components", components, + ) +} + +// PeriodicMetricsLogger periodically logs system and worker metrics +type PeriodicMetricsLogger struct { + ctx context.Context + interval time.Duration + stopChan chan struct{} + metricsLog *MetricsLogger +} + +// NewPeriodicMetricsLogger creates a new PeriodicMetricsLogger +func NewPeriodicMetricsLogger(ctx context.Context, interval time.Duration) *PeriodicMetricsLogger { + return &PeriodicMetricsLogger{ + ctx: ctx, + interval: interval, + stopChan: make(chan struct{}), + metricsLog: NewMetricsLogger(ctx), + } +} + +// Start begins periodic metrics logging +func (p *PeriodicMetricsLogger) Start() { + go func() { + ticker := time.NewTicker(p.interval) + defer ticker.Stop() + + for { + select { + case <-p.stopChan: + return + case <-ticker.C: + p.metricsLog.LogSystemMetrics() + } + } + }() +} + +// Stop stops periodic metrics logging +func (p *PeriodicMetricsLogger) Stop() { + close(p.stopChan) +} \ No newline at end of file diff --git a/task/queue_message.go b/task/queue_message.go new file mode 100644 index 0000000..c87c02c --- /dev/null +++ b/task/queue_message.go @@ -0,0 +1,64 @@ +package task + +import ( + "encoding/json" + + "modelRT/constants" + + "github.com/gofrs/uuid" +) + +// TaskQueueMessage defines minimal message structure for RabbitMQ/Redis queue dispatch +type TaskQueueMessage struct { + TaskID uuid.UUID `json:"task_id"` + TaskType TaskType `json:"task_type"` + Priority int `json:"priority,omitempty"` + TraceCarrier map[string]string `json:"trace_carrier,omitempty"` + Params map[string]any `json:"params,omitempty"` +} + +func NewTaskQueueMessage(taskID uuid.UUID, taskType TaskType) *TaskQueueMessage { + return &TaskQueueMessage{ + TaskID: taskID, + TaskType: taskType, + Priority: constants.TaskPriorityDefault, + } +} + +func NewTaskQueueMessageWithPriority(taskID uuid.UUID, taskType TaskType, priority int) *TaskQueueMessage { + return &TaskQueueMessage{ + TaskID: taskID, + TaskType: taskType, + Priority: priority, + } +} + +func (m *TaskQueueMessage) ToJSON() ([]byte, error) { + return json.Marshal(m) +} + +func (m *TaskQueueMessage) Validate() bool { + if m.TaskID == uuid.Nil { + return false + } + switch m.TaskType { + case TypeTopologyAnalysis, TypeEventAnalysis, TypeBatchImport, TypeTest: + return true + default: + return false + } +} + +func (m *TaskQueueMessage) SetPriority(priority int) { + if priority < constants.TaskPriorityLow { + priority = constants.TaskPriorityLow + } + if priority > constants.TaskPriorityHigh { + priority = constants.TaskPriorityHigh + } + m.Priority = priority +} + +func (m *TaskQueueMessage) GetPriority() int { + return m.Priority +} diff --git a/task/queue_producer.go b/task/queue_producer.go new file mode 100644 index 0000000..3ae6185 --- /dev/null +++ b/task/queue_producer.go @@ -0,0 +1,267 @@ +// Package task provides asynchronous task processing with RabbitMQ integration +package task + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "modelRT/config" + "modelRT/constants" + "modelRT/logger" + "modelRT/mq" + + "github.com/gofrs/uuid" + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" +) + +// TaskMsgChan buffers task messages to be published to RabbitMQ asynchronously +var TaskMsgChan chan *TaskQueueMessage + +func init() { + TaskMsgChan = make(chan *TaskQueueMessage, 10000) +} + +// QueueProducer handles publishing tasks to RabbitMQ +type QueueProducer struct { + conn *amqp.Connection + ch *amqp.Channel +} + +// NewQueueProducer creates a new QueueProducer instance +func NewQueueProducer(ctx context.Context, cfg config.RabbitMQConfig) (*QueueProducer, error) { + // Initialize RabbitMQ connection if not already initialized + mq.InitRabbitProxy(ctx, cfg) + + conn := mq.GetConn() + if conn == nil { + return nil, fmt.Errorf("failed to get RabbitMQ connection") + } + + ch, err := conn.Channel() + if err != nil { + return nil, fmt.Errorf("failed to open channel: %w", err) + } + + producer := &QueueProducer{ + conn: conn, + ch: ch, + } + + // Declare exchange and queue + if err := producer.declareInfrastructure(); err != nil { + ch.Close() + return nil, fmt.Errorf("failed to declare infrastructure: %w", err) + } + + return producer, nil +} + +// declareInfrastructure declares the exchange, queue, and binds them +func (p *QueueProducer) declareInfrastructure() error { + // Declare durable direct exchange + err := p.ch.ExchangeDeclare( + constants.TaskExchangeName, // name + "direct", // type + true, // durable + false, // auto-deleted + false, // internal + false, // no-wait + nil, // arguments + ) + if err != nil { + return fmt.Errorf("failed to declare exchange: %w", err) + } + + // Declare durable queue with priority support and message TTL + _, err = p.ch.QueueDeclare( + constants.TaskQueueName, // name + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait + amqp.Table{ + "x-max-priority": constants.TaskMaxPriority, // support priority levels 0-10 + "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), // message TTL + }, + ) + if err != nil { + return fmt.Errorf("failed to declare queue: %w", err) + } + + // Bind queue to exchange + err = p.ch.QueueBind( + constants.TaskQueueName, // queue name + constants.TaskRoutingKey, // routing key + constants.TaskExchangeName, // exchange name + false, // no-wait + nil, // arguments + ) + if err != nil { + return fmt.Errorf("failed to bind queue: %w", err) + } + + return nil +} + +// PublishTask publishes a task message to RabbitMQ +func (p *QueueProducer) PublishTask(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int, params map[string]any) error { + message := NewTaskQueueMessageWithPriority(taskID, taskType, priority) + message.Params = params + + // Validate message + if !message.Validate() { + return fmt.Errorf("invalid task message: taskID=%s, taskType=%s", taskID, taskType) + } + + // Inject OTel trace context so the consumer (worker) can restore the span chain + carrier := make(map[string]string) + otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(carrier)) + message.TraceCarrier = carrier + + // Convert message to JSON + body, err := json.Marshal(message) + if err != nil { + return fmt.Errorf("failed to marshal task message: %w", err) + } + + // Prepare publishing options + publishing := amqp.Publishing{ + ContentType: "application/json", + Body: body, + DeliveryMode: amqp.Persistent, // Persistent messages survive broker restart + Timestamp: time.Now(), + Priority: uint8(priority), + Headers: amqp.Table{ + "task_id": taskID.String(), + "task_type": string(taskType), + }, + } + + // Publish to exchange + err = p.ch.PublishWithContext( + ctx, + constants.TaskExchangeName, // exchange + constants.TaskRoutingKey, // routing key + false, // mandatory + false, // immediate + publishing, + ) + if err != nil { + return fmt.Errorf("failed to publish task message: %w", err) + } + + logger.Info(ctx, "Task published to queue", + "task_id", taskID.String(), + "task_type", taskType, + "priority", priority, + "queue", constants.TaskQueueName, + ) + + return nil +} + +// PublishTaskWithRetry publishes a task with retry logic +func (p *QueueProducer) PublishTaskWithRetry(ctx context.Context, taskID uuid.UUID, taskType TaskType, priority int, params map[string]any, maxRetries int) error { + var lastErr error + for i := range maxRetries { + err := p.PublishTask(ctx, taskID, taskType, priority, params) + if err == nil { + return nil + } + lastErr = err + + // Exponential backoff + backoff := time.Duration(1< 1 { + randomFactor = 1 + } + + return &ExponentialBackoffRetry{ + MaxRetries: maxRetries, + InitialDelay: initialDelay, + MaxDelay: maxDelay, + RandomFactor: randomFactor, + } +} + +// ShouldRetry implements exponential backoff with jitter +func (s *ExponentialBackoffRetry) ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) { + if retryCount >= s.MaxRetries { + logger.Info(ctx, "Task reached maximum retry count", + "task_id", taskID, + "retry_count", retryCount, + "max_retries", s.MaxRetries, + "last_error", lastError, + ) + return false, 0 + } + + // Calculate exponential backoff: initialDelay * 2^retryCount + delay := min( + // Apply maximum delay cap + s.InitialDelay*time.Duration(math.Pow(2, float64(retryCount))), s.MaxDelay) + + // Add jitter to avoid thundering herd + if s.RandomFactor > 0 { + jitter := rand.Float64() * s.RandomFactor * float64(delay) + // Randomly add or subtract jitter + if rand.Intn(2) == 0 { + delay += time.Duration(jitter) + } else { + delay -= time.Duration(jitter) + } + // Ensure delay doesn't go below initial delay + if delay < s.InitialDelay { + delay = s.InitialDelay + } + } + + logger.Info(ctx, "Task will be retried", + "task_id", taskID, + "retry_count", retryCount, + "next_retry_in", delay, + "max_retries", s.MaxRetries, + ) + + return true, delay +} + +// GetMaxRetries returns maximum number of retry attempts +func (s *ExponentialBackoffRetry) GetMaxRetries() int { + return s.MaxRetries +} + +// FixedDelayRetry implements fixed delay retry strategy +type FixedDelayRetry struct { + MaxRetries int + Delay time.Duration + RandomFactor float64 +} + +// NewFixedDelayRetry creates a new fixed delay retry strategy +func NewFixedDelayRetry(maxRetries int, delay time.Duration, randomFactor float64) *FixedDelayRetry { + if maxRetries < 0 { + maxRetries = 0 + } + if delay <= 0 { + delay = 5 * time.Second + } + + return &FixedDelayRetry{ + MaxRetries: maxRetries, + Delay: delay, + RandomFactor: randomFactor, + } +} + +// ShouldRetry implements fixed delay with optional jitter +func (s *FixedDelayRetry) ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) { + if retryCount >= s.MaxRetries { + return false, 0 + } + + delay := s.Delay + + // Add jitter if random factor is specified + if s.RandomFactor > 0 { + jitter := rand.Float64() * s.RandomFactor * float64(delay) + if rand.Intn(2) == 0 { + delay += time.Duration(jitter) + } else { + delay -= time.Duration(jitter) + } + // Ensure positive delay + if delay <= 0 { + delay = s.Delay + } + } + + return true, delay +} + +// GetMaxRetries returns maximum number of retry attempts +func (s *FixedDelayRetry) GetMaxRetries() int { + return s.MaxRetries +} + +// NoRetryStrategy implements a strategy that never retries +type NoRetryStrategy struct{} + +// NewNoRetryStrategy creates a new no-retry strategy +func NewNoRetryStrategy() *NoRetryStrategy { + return &NoRetryStrategy{} +} + +// ShouldRetry always returns false +func (s *NoRetryStrategy) ShouldRetry(ctx context.Context, taskID string, retryCount int, lastError error) (bool, time.Duration) { + return false, 0 +} + +// GetMaxRetries returns 0 +func (s *NoRetryStrategy) GetMaxRetries() int { + return 0 +} + +// DefaultRetryStrategy returns the default retry strategy (exponential backoff) +func DefaultRetryStrategy() RetryStrategy { + return NewExponentialBackoffRetry( + constants.TaskRetryMaxDefault, // max retries + constants.TaskRetryInitialDelayDefault, // initial delay + constants.TaskRetryMaxDelayDefault, // max delay + constants.TaskRetryRandomFactorDefault, // random factor (10% jitter) + ) +} + +// IsRetryableError checks if an error is retryable based on common patterns +func IsRetryableError(err error) bool { + if err == nil { + return false + } + + errorMsg := err.Error() + + // Check for transient errors that are typically retryable + retryablePatterns := []string{ + "timeout", + "deadline exceeded", + "temporary", + "busy", + "connection refused", + "connection reset", + "network", + "too many connections", + "resource temporarily unavailable", + "rate limit", + "throttle", + "server unavailable", + "service unavailable", + } + + for _, pattern := range retryablePatterns { + if strings.Contains(strings.ToLower(errorMsg), pattern) { + return true + } + } + + return false +} diff --git a/task/retry_queue.go b/task/retry_queue.go new file mode 100644 index 0000000..34499bd --- /dev/null +++ b/task/retry_queue.go @@ -0,0 +1,187 @@ +// Package task provides retry queue management for failed asynchronous tasks +package task + +import ( + "context" + "time" + + "modelRT/database" + "modelRT/logger" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// RetryQueue manages scheduling and execution of task retries +type RetryQueue struct { + db *gorm.DB + producer *QueueProducer + strategy RetryStrategy +} + +// NewRetryQueue creates a new RetryQueue instance +func NewRetryQueue(db *gorm.DB, producer *QueueProducer, strategy RetryStrategy) *RetryQueue { + if strategy == nil { + strategy = DefaultRetryStrategy() + } + + return &RetryQueue{ + db: db, + producer: producer, + strategy: strategy, + } +} + +// ScheduleRetry schedules a failed task for retry based on retry strategy +func (q *RetryQueue) ScheduleRetry(ctx context.Context, taskID uuid.UUID, taskType TaskType, retryCount int, lastError error) error { + // Check if task should be retried + shouldRetry, delay := q.strategy.ShouldRetry(ctx, taskID.String(), retryCount, lastError) + if !shouldRetry { + // Mark task as permanently failed + logger.Info(ctx, "Task will not be retried, marking as failed", + "task_id", taskID, + "retry_count", retryCount, + "max_retries", q.strategy.GetMaxRetries(), + "last_error", lastError, + ) + return database.FailAsyncTask(ctx, q.db, taskID, time.Now().Unix()) + } + + // Calculate next retry time + nextRetryTime := time.Now().Add(delay).Unix() + + // Update task retry information in database + err := q.db.Transaction(func(tx *gorm.DB) error { + if err := database.UpdateTaskRetryInfo(ctx, tx, taskID, retryCount+1, nextRetryTime); err != nil { + return err + } + + // Update error information + errorMsg := "" + if lastError != nil { + errorMsg = lastError.Error() + } + if err := database.UpdateTaskErrorInfo(ctx, tx, taskID, errorMsg, ""); err != nil { + // Log but don't fail the whole retry scheduling + logger.Warn(ctx, "Failed to update task error info", + "task_id", taskID, + "error", err, + ) + } + + // Task will be picked up by ProcessRetryQueue when next_retry_time is reached + return nil + }) + + if err != nil { + logger.Error(ctx, "Failed to schedule task retry", + "task_id", taskID, + "task_type", taskType, + "retry_count", retryCount, + "delay", delay, + "error", err, + ) + return err + } + + logger.Info(ctx, "Task scheduled for retry", + "task_id", taskID, + "task_type", taskType, + "retry_count", retryCount+1, + "next_retry_in", delay, + "next_retry_time", time.Unix(nextRetryTime, 0).Format(time.RFC3339), + ) + + return nil +} + +// ProcessRetryQueue processes tasks that are due for retry +func (q *RetryQueue) ProcessRetryQueue(ctx context.Context, batchSize int) error { + // Get tasks due for retry + tasks, err := database.GetTasksForRetry(ctx, q.db, batchSize) + if err != nil { + logger.Error(ctx, "Failed to get tasks for retry", "error", err) + return err + } + + if len(tasks) == 0 { + return nil + } + + logger.Info(ctx, "Processing retry queue", + "task_count", len(tasks), + "batch_size", batchSize, + ) + + for _, task := range tasks { + select { + case <-ctx.Done(): + return ctx.Err() + default: + // Publish task to queue for immediate processing + taskType := TaskType(task.TaskType) + if err := q.producer.PublishTask(ctx, task.TaskID, taskType, task.Priority, map[string]any(task.Params)); err != nil { + logger.Error(ctx, "Failed to publish retry task to queue", + "task_id", task.TaskID, + "task_type", taskType, + "error", err, + ) + // Continue with other tasks + continue + } + + // Update task status back to submitted + if err := database.UpdateAsyncTaskStatus(ctx, q.db, task.TaskID, "SUBMITTED"); err != nil { + logger.Warn(ctx, "Failed to update retry task status", + "task_id", task.TaskID, + "error", err, + ) + } + + // Clear next retry time since task is being retried now + if err := database.UpdateTaskRetryInfo(ctx, q.db, task.TaskID, task.RetryCount, 0); err != nil { + logger.Warn(ctx, "Failed to clear next retry time", + "task_id", task.TaskID, + "error", err, + ) + } + + logger.Info(ctx, "Retry task resubmitted", + "task_id", task.TaskID, + "task_type", taskType, + "retry_count", task.RetryCount, + ) + } + } + + return nil +} + +// StartRetryScheduler starts a background goroutine to periodically process retry queue +func (q *RetryQueue) StartRetryScheduler(ctx context.Context, interval time.Duration, batchSize int) { + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + logger.Info(ctx, "Retry scheduler stopping") + return + case <-ticker.C: + if err := q.ProcessRetryQueue(ctx, batchSize); err != nil { + logger.Error(ctx, "Error processing retry queue", "error", err) + } + } + } + }() +} + +// GetRetryStats returns statistics about retry queue +func (q *RetryQueue) GetRetryStats(ctx context.Context) (int, error) { + tasks, err := database.GetTasksForRetry(ctx, q.db, 1000) // Large limit to count + if err != nil { + return 0, err + } + return len(tasks), nil +} \ No newline at end of file diff --git a/task/test_task.go b/task/test_task.go new file mode 100644 index 0000000..14edb14 --- /dev/null +++ b/task/test_task.go @@ -0,0 +1,165 @@ +// Package task provides test task implementation for system verification +package task + +import ( + "context" + "fmt" + "time" + + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/orm" + + "github.com/gofrs/uuid" + "gorm.io/gorm" +) + +// TestTaskParams defines parameters for test task +type TestTaskParams struct { + // SleepDuration specifies how long the task should sleep (in seconds) + // Default is constants.TestTaskSleepDurationDefault seconds as per requirement + SleepDuration int `json:"sleep_duration"` + // Message is a custom message to include in the result + Message string `json:"message,omitempty"` +} + +// Validate checks if test task parameters are valid +func (p *TestTaskParams) Validate() error { + // Default to constants.TestTaskSleepDurationDefault seconds if not specified + if p.SleepDuration <= 0 { + p.SleepDuration = constants.TestTaskSleepDurationDefault + } + + // Validate max duration (max 1 hour) + if p.SleepDuration > constants.TestTaskSleepDurationMax { + return fmt.Errorf("sleep duration cannot exceed %d seconds (1 hour)", constants.TestTaskSleepDurationMax) + } + + return nil +} + +// GetType returns the task type +func (p *TestTaskParams) GetType() UnifiedTaskType { + return TaskTypeTest +} + +// ToMap converts parameters to map for database storage +func (p *TestTaskParams) ToMap() map[string]any { + return map[string]any{ + "sleep_duration": p.SleepDuration, + "message": p.Message, + } +} + +// FromMap populates parameters from map (for database retrieval) +func (p *TestTaskParams) FromMap(params map[string]any) error { + if v, ok := params["sleep_duration"]; ok { + if duration, isFloat := v.(float64); isFloat { + p.SleepDuration = int(duration) + } else if duration, isInt := v.(int); isInt { + p.SleepDuration = duration + } + } + + if v, ok := params["message"]; ok { + if msg, isString := v.(string); isString { + p.Message = msg + } + } + + return nil +} + +// TestTask implements a test task that sleeps for specified duration +// This task contains no I/O operations as per requirements +type TestTask struct { + *BaseTask +} + +// NewTestTask creates a new TestTask instance +func NewTestTask(params TestTaskParams) *TestTask { + return &TestTask{ + BaseTask: NewBaseTask(TaskTypeTest, ¶ms, "test_task"), + } +} + +// Execute performs the test task logic (sleep without I/O operations) +func (t *TestTask) Execute(ctx context.Context, taskID uuid.UUID, db *gorm.DB) error { + params, ok := t.GetParams().(*TestTaskParams) + if !ok { + return fmt.Errorf("invalid parameter type for TestTask") + } + + logger.Info(ctx, "Starting test task executionser", + "task_id", taskID, + "sleep_duration_seconds", params.SleepDuration, + "message", params.Message, + ) + + // Sleep for the specified duration without any I/O operations + // This is pure CPU-time wait as per requirements + sleepDuration := time.Duration(params.SleepDuration) * time.Second + time.Sleep(sleepDuration) + + // Build result + result := map[string]any{ + "status": "completed", + "sleep_duration": params.SleepDuration, + "message": params.Message, + "executed_at": time.Now().Unix(), + "task_id": taskID.String(), + } + + // Save result to database + if err := database.UpdateAsyncTaskResultWithSuccess(ctx, db, taskID, orm.JSONMap(result)); err != nil { + logger.Error(ctx, "Failed to save test task result", + "task_id", taskID, + "error", err, + ) + return fmt.Errorf("failed to save task result: %w", err) + } + + logger.Info(ctx, "Test task completed successfully", + "task_id", taskID, + "sleep_duration_seconds", params.SleepDuration, + ) + + return nil +} + +// TestTaskHandler handles test task execution +type TestTaskHandler struct { + BaseHandler +} + +// NewTestTaskHandler creates a new TestTaskHandler +func NewTestTaskHandler() *TestTaskHandler { + return &TestTaskHandler{ + BaseHandler: *NewBaseHandler("test_task_handler"), + } +} + +// Execute processes a test task using the unified task interface +func (h *TestTaskHandler) Execute(ctx context.Context, taskID uuid.UUID, params map[string]any, db *gorm.DB) error { + logger.Info(ctx, "Executing test task", + "task_id", taskID, + "task_params", params, + "db", db, + ) + + // Convert params from MQ message to TestTaskParams + taskParams := &TestTaskParams{} + if err := taskParams.FromMap(params); err != nil { + return fmt.Errorf("failed to parse task params: %w", err) + } + + // Create and execute test task + testTask := NewTestTask(*taskParams) + return testTask.Execute(ctx, taskID, db) +} + +// CanHandle returns true for test tasks +func (h *TestTaskHandler) CanHandle(taskType TaskType) bool { + return string(TaskTypeTest) == string(taskType) +} diff --git a/task/types.go b/task/types.go new file mode 100644 index 0000000..ef59f6c --- /dev/null +++ b/task/types.go @@ -0,0 +1,32 @@ +package task + +// TaskType defines the business type of an async task +type TaskType string + +const ( + TypeTopologyAnalysis TaskType = "TOPOLOGY_ANALYSIS" + TypeEventAnalysis TaskType = "EVENT_ANALYSIS" + TypeBatchImport TaskType = "BATCH_IMPORT" + TypeTest TaskType = "TEST" +) + +// TaskStatus defines the lifecycle status of an async task +type TaskStatus string + +const ( + StatusPending TaskStatus = "PENDING" + StatusRunning TaskStatus = "RUNNING" + StatusCompleted TaskStatus = "COMPLETED" + StatusFailed TaskStatus = "FAILED" +) + +// UnifiedTaskType defines all async task types in a single location +type UnifiedTaskType string + +const ( + TaskTypeTopologyAnalysis UnifiedTaskType = "TOPOLOGY_ANALYSIS" + TaskTypePerformanceAnalysis UnifiedTaskType = "PERFORMANCE_ANALYSIS" + TaskTypeEventAnalysis UnifiedTaskType = "EVENT_ANALYSIS" + TaskTypeBatchImport UnifiedTaskType = "BATCH_IMPORT" + TaskTypeTest UnifiedTaskType = "TEST" +) diff --git a/task/worker.go b/task/worker.go new file mode 100644 index 0000000..d60c1a3 --- /dev/null +++ b/task/worker.go @@ -0,0 +1,642 @@ +// Package task provides asynchronous task processing with worker pools +package task + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "sync" + "time" + + "modelRT/config" + "modelRT/constants" + "modelRT/database" + "modelRT/logger" + "modelRT/mq" + "modelRT/mq/event" + "modelRT/orm" + + "github.com/gofrs/uuid" + "github.com/panjf2000/ants/v2" + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" + "gorm.io/gorm" +) + +// WorkerConfig holds configuration for the task worker +type WorkerConfig struct { + // PoolSize is the number of worker goroutines in the pool + PoolSize int + // PreAlloc indicates whether to pre-allocate memory for the pool + PreAlloc bool + // MaxBlockingTasks is the maximum number of tasks waiting in queue + MaxBlockingTasks int + // QueueConsumerCount is the number of concurrent RabbitMQ consumers + QueueConsumerCount int + // PollingInterval is the interval between health checks + PollingInterval time.Duration +} + +// DefaultWorkerConfig returns the default worker configuration +func DefaultWorkerConfig() WorkerConfig { + return WorkerConfig{ + PoolSize: 10, + PreAlloc: true, + MaxBlockingTasks: 100, + QueueConsumerCount: 2, + PollingInterval: 30 * time.Second, + } +} + +// TaskWorker manages a pool of workers for processing asynchronous tasks +type TaskWorker struct { + cfg WorkerConfig + db *gorm.DB + pool *ants.Pool + conn *amqp.Connection + ch *amqp.Channel + factory *HandlerFactory + retryQueue *RetryQueue + stopChan chan struct{} + wg sync.WaitGroup + ctx context.Context + cancel context.CancelFunc + metrics *WorkerMetrics +} + +// WorkerMetrics holds metrics for the worker pool +type WorkerMetrics struct { + // Task statistics by type + TasksProcessed map[TaskType]int64 + TasksFailed map[TaskType]int64 + TasksSuccess map[TaskType]int64 + ProcessingTime map[TaskType]time.Duration + + // Aggregate counters (maintained for backward compatibility) + TotalProcessed int64 + TotalFailed int64 + TotalSuccess int64 + TasksInProgress int32 + + // Queue and latency metrics + QueueDepth int + QueueLatency time.Duration + + // Worker resource metrics + WorkersActive int + WorkersIdle int + MemoryUsage uint64 + CPULoad float64 + + // Time window metrics + LastMinuteRate float64 + Last5MinutesRate float64 + LastHourRate float64 + + // Health and timing + LastHealthCheck time.Time + mu sync.RWMutex +} + +// NewTaskWorker creates a new TaskWorker instance +func NewTaskWorker(ctx context.Context, cfg WorkerConfig, db *gorm.DB, rabbitCfg config.RabbitMQConfig, factory *HandlerFactory) (*TaskWorker, error) { + // Initialize RabbitMQ connection + mq.InitRabbitProxy(ctx, rabbitCfg) + conn := mq.GetConn() + if conn == nil { + return nil, fmt.Errorf("failed to get RabbitMQ connection") + } + + // Create channel + ch, err := conn.Channel() + if err != nil { + return nil, fmt.Errorf("failed to open channel: %w", err) + } + + // Declare queue (ensure it exists with proper arguments) + _, err = ch.QueueDeclare( + constants.TaskQueueName, // name + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait + amqp.Table{ + "x-max-priority": constants.TaskMaxPriority, + "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), + }, + ) + if err != nil { + ch.Close() + return nil, fmt.Errorf("failed to declare queue: %w", err) + } + + // Set QoS (quality of service) for fair dispatch + err = ch.Qos( + 1, // prefetch count + 0, // prefetch size + false, // global + ) + if err != nil { + ch.Close() + return nil, fmt.Errorf("failed to set QoS: %w", err) + } + + // Create ants pool + pool, err := ants.NewPool(cfg.PoolSize, ants.WithPreAlloc(cfg.PreAlloc)) + if err != nil { + ch.Close() + return nil, fmt.Errorf("failed to create worker pool: %w", err) + } + + ctxWithCancel, cancel := context.WithCancel(ctx) + + worker := &TaskWorker{ + cfg: cfg, + db: db, + pool: pool, + conn: conn, + ch: ch, + factory: factory, + stopChan: make(chan struct{}), + ctx: ctxWithCancel, + cancel: cancel, + metrics: &WorkerMetrics{ + TasksProcessed: make(map[TaskType]int64), + TasksFailed: make(map[TaskType]int64), + TasksSuccess: make(map[TaskType]int64), + ProcessingTime: make(map[TaskType]time.Duration), + LastHealthCheck: time.Now(), + }, + } + + return worker, nil +} + +// Start begins consuming tasks from the queue +func (w *TaskWorker) Start() error { + logger.Info(w.ctx, "Starting task worker", + "pool_size", w.cfg.PoolSize, + "queue_consumers", w.cfg.QueueConsumerCount, + ) + + // Start multiple consumers for better throughput + for i := 0; i < w.cfg.QueueConsumerCount; i++ { + w.wg.Add(1) + go w.consumerLoop(i) + } + + // Start health check goroutine + w.wg.Add(1) + go w.healthCheckLoop() + + logger.Info(w.ctx, "Task worker started successfully") + return nil +} + +// consumerLoop runs a single RabbitMQ consumer +func (w *TaskWorker) consumerLoop(consumerID int) { + defer w.wg.Done() + + logger.Info(w.ctx, "Starting consumer", "consumer_id", consumerID) + + // Consume messages from the queue + msgs, err := w.ch.Consume( + constants.TaskQueueName, // queue + fmt.Sprintf("worker-%d", consumerID), // consumer tag + false, // auto-ack + false, // exclusive + false, // no-local + false, // no-wait + nil, // args + ) + if err != nil { + logger.Error(w.ctx, "Failed to start consumer", + "consumer_id", consumerID, + "error", err, + ) + return + } + + for { + select { + case <-w.stopChan: + logger.Info(w.ctx, "Consumer stopping", "consumer_id", consumerID) + return + case msg, ok := <-msgs: + if !ok { + logger.Warn(w.ctx, "Consumer channel closed", "consumer_id", consumerID) + return + } + + // Process message in worker pool + err := w.pool.Submit(func() { + w.handleMessage(msg) + }) + if err != nil { + logger.Error(w.ctx, "Failed to submit task to pool", + "consumer_id", consumerID, + "error", err, + ) + // Reject message and requeue + msg.Nack(false, true) + } + } + } +} + +// handleMessage processes a single RabbitMQ message +func (w *TaskWorker) handleMessage(msg amqp.Delivery) { + w.metrics.mu.Lock() + w.metrics.TasksInProgress++ + w.metrics.mu.Unlock() + + defer func() { + w.metrics.mu.Lock() + w.metrics.TasksInProgress-- + w.metrics.mu.Unlock() + }() + + ctx := w.ctx + + // Parse task message + var taskMsg TaskQueueMessage + if err := json.Unmarshal(msg.Body, &taskMsg); err != nil { + logger.Error(ctx, "Failed to unmarshal task message", "error", err) + msg.Nack(false, false) // Reject without requeue + w.metrics.mu.Lock() + w.metrics.TotalFailed++ + w.metrics.mu.Unlock() + return + } + + // Validate message + if !taskMsg.Validate() { + logger.Error(ctx, "Invalid task message", + "task_id", taskMsg.TaskID, + "task_type", taskMsg.TaskType, + ) + msg.Nack(false, false) // Reject without requeue + w.metrics.mu.Lock() + w.metrics.TotalFailed++ + // Also update per-task-type failure count + w.metrics.TasksFailed[taskMsg.TaskType]++ + w.metrics.mu.Unlock() + return + } + + // Restore trace context from the publish span, then create an execute child span + taskCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(taskMsg.TraceCarrier)) + taskCtx, span := otel.Tracer("modelRT/task").Start(taskCtx, "task.execute", + oteltrace.WithAttributes( + attribute.String("task_id", taskMsg.TaskID.String()), + attribute.String("task_type", string(taskMsg.TaskType)), + ), + ) + defer span.End() + ctx = taskCtx + + logger.Info(ctx, "Processing task", + "task_id", taskMsg.TaskID, + "task_type", taskMsg.TaskType, + "priority", taskMsg.Priority, + ) + + // Update task status to RUNNING in database + if err := w.updateTaskStatus(ctx, taskMsg.TaskID, StatusRunning); err != nil { + logger.Error(ctx, "Failed to update task status", "error", err) + msg.Nack(false, true) // Reject with requeue + w.metrics.mu.Lock() + w.metrics.TotalFailed++ + w.metrics.TasksFailed[taskMsg.TaskType]++ + w.metrics.mu.Unlock() + return + } + + if record, err := event.NewTaskRunningMessage(taskMsg.TaskID.String(), string(taskMsg.TaskType)); err == nil { + mq.TryEmitMessage(ctx, record) + } + + // Execute task using handler + startTime := time.Now() + err := w.dispatch(ctx, taskMsg.TaskType, taskMsg.TaskID, taskMsg.Params, &msg) + processingTime := time.Since(startTime) + + if err != nil { + logger.Error(ctx, "Task execution failed", + "task_id", taskMsg.TaskID, + "task_type", taskMsg.TaskType, + "processing_time", processingTime, + "error", err, + ) + + // Update task status to FAILED + if updateErr := w.updateTaskWithError(ctx, taskMsg.TaskID, err); updateErr != nil { + logger.Error(ctx, "Failed to update task with error", "error", updateErr) + } + + if record, recErr := event.NewTaskFailedMessage(taskMsg.TaskID.String(), string(taskMsg.TaskType), err.Error()); recErr == nil { + mq.TryEmitMessage(ctx, record) + } + + // Ack message even if task failed (we don't want to retry indefinitely) + msg.Ack(false) + w.metrics.mu.Lock() + w.metrics.TotalFailed++ + w.metrics.TasksFailed[taskMsg.TaskType]++ + w.metrics.mu.Unlock() + return + } + + // Update task status to COMPLETED + if err := w.updateTaskStatus(ctx, taskMsg.TaskID, StatusCompleted); err != nil { + logger.Error(ctx, "Failed to update task status to completed", "error", err) + // Still ack the message since task was processed successfully + } + + if record, err := event.NewTaskCompletedMessage(taskMsg.TaskID.String(), string(taskMsg.TaskType), processingTime.Milliseconds()); err == nil { + mq.TryEmitMessage(ctx, record) + } + + // Acknowledge message + msg.Ack(false) + + logger.Info(ctx, "Task completed successfully", + "task_id", taskMsg.TaskID, + "task_type", taskMsg.TaskType, + "processing_time", processingTime, + ) + + w.metrics.mu.Lock() + w.metrics.TotalProcessed++ + w.metrics.TasksProcessed[taskMsg.TaskType]++ + w.metrics.TasksSuccess[taskMsg.TaskType]++ + w.metrics.ProcessingTime[taskMsg.TaskType] += processingTime + w.metrics.mu.Unlock() +} + +// updateTaskStatus updates the status of a task in the database +func (w *TaskWorker) updateTaskStatus(ctx context.Context, taskID uuid.UUID, status TaskStatus) error { + // Convert TaskStatus to orm.AsyncTaskStatus + var ormStatus orm.AsyncTaskStatus + switch status { + case StatusPending: + ormStatus = orm.AsyncTaskStatusSubmitted + case StatusRunning: + ormStatus = orm.AsyncTaskStatusRunning + case StatusCompleted: + ormStatus = orm.AsyncTaskStatusCompleted + case StatusFailed: + ormStatus = orm.AsyncTaskStatusFailed + default: + return fmt.Errorf("unknown task status: %s", status) + } + + // Update task status in database + err := database.UpdateAsyncTaskStatus(ctx, w.db, taskID, ormStatus) + if err != nil { + logger.Error(ctx, "Failed to update task status in database", + "task_id", taskID, + "status", status, + "error", err, + ) + return err + } + + // If status is running, update started_at timestamp + if status == StatusRunning { + startedAt := time.Now().Unix() + if err := database.UpdateTaskStarted(ctx, w.db, taskID, startedAt); err != nil { + logger.Warn(ctx, "Failed to update task start time", + "task_id", taskID, + "error", err, + ) + // Continue despite error + } + } + + // If status is completed or failed, update finished_at timestamp + if status == StatusCompleted || status == StatusFailed { + finishedAt := time.Now().Unix() + if status == StatusCompleted { + if err := database.CompleteAsyncTask(ctx, w.db, taskID, finishedAt); err != nil { + logger.Warn(ctx, "Failed to mark task as completed", + "task_id", taskID, + "error", err, + ) + } + } else { + if err := database.FailAsyncTask(ctx, w.db, taskID, finishedAt); err != nil { + logger.Warn(ctx, "Failed to mark task as failed", + "task_id", taskID, + "error", err, + ) + } + } + } + + logger.Debug(ctx, "Task status updated", + "task_id", taskID, + "status", status, + ) + return nil +} + +// updateTaskWithError updates a task with error information in both async_task and async_task_result. +func (w *TaskWorker) updateTaskWithError(ctx context.Context, taskID uuid.UUID, err error) error { + errorMsg := err.Error() + stackTrace := fmt.Sprintf("%+v", err) + + if updateErr := database.UpdateTaskErrorInfo(ctx, w.db, taskID, errorMsg, stackTrace); updateErr != nil { + logger.Error(ctx, "Failed to update task error info", "task_id", taskID, "error", updateErr) + return updateErr + } + + if updateErr := database.UpdateAsyncTaskResultWithError(ctx, w.db, taskID, 500, errorMsg, nil); updateErr != nil { + logger.Error(ctx, "Failed to update task result with error", "task_id", taskID, "error", updateErr) + return updateErr + } + + logger.Warn(ctx, "Task failed with error", "task_id", taskID, "error", errorMsg) + return nil +} + +// dispatch routes a task message to the appropriate handler and executes it. +// Nacks the message and returns an error if no handler is registered for the task type. +func (w *TaskWorker) dispatch(ctx context.Context, taskType TaskType, taskID uuid.UUID, params map[string]any, msg *amqp.Delivery) error { + handler, err := w.factory.GetHandler(taskType) + if err != nil { + logger.Error(ctx, "No handler for task type", "task_type", taskType) + msg.Nack(false, false) + return err + } + return handler.Execute(ctx, taskID, params, w.db) +} + +// healthCheckLoop periodically checks worker health and metrics +func (w *TaskWorker) healthCheckLoop() { + defer w.wg.Done() + + ticker := time.NewTicker(w.cfg.PollingInterval) + defer ticker.Stop() + + for { + select { + case <-w.stopChan: + return + case <-ticker.C: + w.checkHealth() + } + } +} + +// checkHealth performs health checks and logs metrics +func (w *TaskWorker) checkHealth() { + w.metrics.mu.Lock() + defer w.metrics.mu.Unlock() + + // Update queue depth + queue, err := w.ch.QueueDeclarePassive( + constants.TaskQueueName, // name + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait + amqp.Table{ + "x-max-priority": constants.TaskMaxPriority, + "x-message-ttl": constants.TaskDefaultMessageTTL.Milliseconds(), + }, + ) + if err == nil { + w.metrics.QueueDepth = queue.Messages + } + + // Update worker pool stats + w.metrics.WorkersActive = w.pool.Running() + w.metrics.WorkersIdle = w.pool.Free() + w.metrics.LastHealthCheck = time.Now() + + logger.Info(w.ctx, "Worker health check", + "tasks_processed", w.metrics.TotalProcessed, + "tasks_failed", w.metrics.TotalFailed, + "tasks_success", w.metrics.TotalSuccess, + "tasks_in_progress", w.metrics.TasksInProgress, + "queue_depth", w.metrics.QueueDepth, + "queue_latency_ms", w.metrics.QueueLatency.Milliseconds(), + "workers_active", w.metrics.WorkersActive, + "workers_idle", w.metrics.WorkersIdle, + "memory_usage_mb", w.metrics.MemoryUsage/(1024*1024), + "cpu_load_percent", w.metrics.CPULoad, + "pool_capacity", w.pool.Cap(), + ) +} + +// Stop gracefully stops the task worker +func (w *TaskWorker) Stop() error { + logger.Info(w.ctx, "Stopping task worker") + + // Signal all goroutines to stop + close(w.stopChan) + w.cancel() + + // Wait for all goroutines to finish + w.wg.Wait() + + // Release worker pool + w.pool.Release() + + // Close channel + if w.ch != nil { + if err := w.ch.Close(); err != nil { + logger.Error(w.ctx, "failed to close channel", "error", err) + } + } + + logger.Info(w.ctx, "task worker stopped") + return nil +} + +// GetMetrics returns current worker metrics +func (w *TaskWorker) GetMetrics() *WorkerMetrics { + w.metrics.mu.RLock() + defer w.metrics.mu.RUnlock() + + // Deep copy maps to avoid data races + tasksProcessedCopy := make(map[TaskType]int64) + maps.Copy(tasksProcessedCopy, w.metrics.TasksProcessed) + + tasksFailedCopy := make(map[TaskType]int64) + maps.Copy(tasksFailedCopy, w.metrics.TasksFailed) + + tasksSuccessCopy := make(map[TaskType]int64) + maps.Copy(tasksSuccessCopy, w.metrics.TasksSuccess) + + processingTimeCopy := make(map[TaskType]time.Duration) + maps.Copy(processingTimeCopy, w.metrics.ProcessingTime) + + // Create a copy without the mutex to avoid copylocks warning + return &WorkerMetrics{ + TasksProcessed: tasksProcessedCopy, + TasksFailed: tasksFailedCopy, + TasksSuccess: tasksSuccessCopy, + ProcessingTime: processingTimeCopy, + TotalProcessed: w.metrics.TotalProcessed, + TotalFailed: w.metrics.TotalFailed, + TotalSuccess: w.metrics.TotalSuccess, + TasksInProgress: w.metrics.TasksInProgress, + QueueDepth: w.metrics.QueueDepth, + QueueLatency: w.metrics.QueueLatency, + WorkersActive: w.metrics.WorkersActive, + WorkersIdle: w.metrics.WorkersIdle, + MemoryUsage: w.metrics.MemoryUsage, + CPULoad: w.metrics.CPULoad, + LastMinuteRate: w.metrics.LastMinuteRate, + Last5MinutesRate: w.metrics.Last5MinutesRate, + LastHourRate: w.metrics.LastHourRate, + LastHealthCheck: w.metrics.LastHealthCheck, + // Mutex is intentionally omitted + } +} + +// IsHealthy returns true if the worker is healthy +func (w *TaskWorker) IsHealthy() bool { + w.metrics.mu.RLock() + defer w.metrics.mu.RUnlock() + + // Consider unhealthy if last health check was too long ago + return time.Since(w.metrics.LastHealthCheck) < 2*w.cfg.PollingInterval +} + +// RecordMetrics periodically records worker metrics to the logging system +func (w *TaskWorker) RecordMetrics(interval time.Duration) { + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + metricsLogger := NewMetricsLogger(w.ctx) + + for { + select { + case <-w.stopChan: + return + case <-ticker.C: + w.metrics.mu.RLock() + metricsLogger.LogWorkerMetrics( + w.metrics.WorkersActive, + w.metrics.WorkersIdle, + w.pool.Cap(), + w.metrics.MemoryUsage, + w.metrics.CPULoad, + ) + metricsLogger.LogQueueMetrics( + w.metrics.QueueDepth, + w.metrics.QueueLatency, + ) + w.metrics.mu.RUnlock() + } + } + }() +} diff --git a/util/redis_init.go b/util/redis_init.go index 8ea504a..f2c61a2 100644 --- a/util/redis_init.go +++ b/util/redis_init.go @@ -3,6 +3,7 @@ package util import ( "context" + "errors" "fmt" "time" @@ -13,25 +14,36 @@ import ( ) // NewRedisClient define func of initialize the Redis client -func NewRedisClient(addr string, opts ...RedisOption) (*redis.Client, error) { +func NewRedisClient(addr string, opts ...Option) (*redis.Client, error) { // default options - options := RedisOptions{ - redisOptions: &redis.Options{ - Addr: addr, + configs := &clientConfig{ + Options: &redis.Options{ + Addr: addr, + DialTimeout: 5 * time.Second, + ReadTimeout: 3 * time.Second, + WriteTimeout: 3 * time.Second, + PoolSize: 10, }, } // Apply configuration options from config + var errs []error for _, opt := range opts { - opt(&options) + if err := opt(configs); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to apply options: %w", errors.Join(errs...)) } // create redis client - client := redis.NewClient(options.redisOptions) + client := redis.NewClient(configs.Options) - if options.timeout > 0 { + if configs.DialTimeout > 0 { // check if the connection is successful - ctx, cancel := context.WithTimeout(context.Background(), options.timeout) + ctx, cancel := context.WithTimeout(context.Background(), configs.DialTimeout) defer cancel() if err := client.Ping(ctx).Err(); err != nil { return nil, fmt.Errorf("can not connect redis:%v", err) @@ -43,22 +55,29 @@ func NewRedisClient(addr string, opts ...RedisOption) (*redis.Client, error) { // NewRedigoPool define func of initialize the Redigo pool func NewRedigoPool(rCfg config.RedisConfig) (*redigo.Pool, error) { pool := &redigo.Pool{ - MaxIdle: rCfg.PoolSize / 2, - MaxActive: rCfg.PoolSize, // TODO optimize IdleTimeout with config parameter + MaxIdle: rCfg.PoolSize / 2, + MaxActive: rCfg.PoolSize, IdleTimeout: 240 * time.Second, + TestOnBorrow: func(c redigo.Conn, t time.Time) error { + if time.Since(t) < time.Minute { + return nil + } + _, err := c.Do("PING") + return err + }, - // Dial function to create the connection Dial: func() (redigo.Conn, error) { - timeout := time.Duration(rCfg.Timeout) * time.Millisecond // 假设 rCfg.Timeout 是毫秒 + dialTimeout := time.Duration(rCfg.DialTimeout) * time.Second + readTimeout := time.Duration(rCfg.ReadTimeout) * time.Second + writeTimeout := time.Duration(rCfg.WriteTimeout) * time.Second + opts := []redigo.DialOption{ redigo.DialDatabase(rCfg.DB), redigo.DialPassword(rCfg.Password), - redigo.DialConnectTimeout(timeout), - // redigo.DialReadTimeout(timeout), - // redigo.DialWriteTimeout(timeout), - // TODO add redigo.DialUsername when redis open acl - // redis.DialUsername("username"), + redigo.DialConnectTimeout(dialTimeout), + redigo.DialReadTimeout(readTimeout), + redigo.DialWriteTimeout(writeTimeout), } c, err := redigo.Dial("tcp", rCfg.Addr, opts...) @@ -72,13 +91,14 @@ func NewRedigoPool(rCfg config.RedisConfig) (*redigo.Pool, error) { conn := pool.Get() defer conn.Close() - if conn.Err() != nil { - return nil, fmt.Errorf("failed to get connection from pool: %w", conn.Err()) + if err := conn.Err(); err != nil { + return nil, fmt.Errorf("failed to get connection from pool: %w", err) } _, err := conn.Do("PING") if err != nil { return nil, fmt.Errorf("redis connection test (PING) failed: %w", err) } + return pool, nil } diff --git a/util/redis_options.go b/util/redis_options.go index dbb4feb..6980ecd 100644 --- a/util/redis_options.go +++ b/util/redis_options.go @@ -5,56 +5,79 @@ import ( "errors" "time" + "modelRT/constants" + "github.com/redis/go-redis/v9" ) -type RedisOptions struct { - redisOptions *redis.Options - timeout time.Duration +type clientConfig struct { + *redis.Options } -type RedisOption func(*RedisOptions) error +type Option func(*clientConfig) error // WithPassword define func of configure redis password options -func WithPassword(password string) RedisOption { - return func(o *RedisOptions) error { - if password == "" { +func WithPassword(password string, env string) Option { + return func(c *clientConfig) error { + if env == constants.ProductionDeployMode && password == "" { return errors.New("password is empty") } - o.redisOptions.Password = password + c.Password = password return nil } } -// WithTimeout define func of configure redis timeout options -func WithTimeout(timeout time.Duration) RedisOption { - return func(o *RedisOptions) error { +// WithConnectTimeout define func of configure redis connect timeout options +func WithConnectTimeout(timeout time.Duration) Option { + return func(c *clientConfig) error { if timeout < 0 { return errors.New("timeout can not be negative") } - o.timeout = timeout + c.DialTimeout = timeout + return nil + } +} + +// WithReadTimeout define func of configure redis read timeout options +func WithReadTimeout(timeout time.Duration) Option { + return func(c *clientConfig) error { + if timeout < 0 { + return errors.New("timeout can not be negative") + } + c.ReadTimeout = timeout + return nil + } +} + +// WithWriteTimeout define func of configure redis write timeout options +func WithWriteTimeout(timeout time.Duration) Option { + return func(c *clientConfig) error { + if timeout < 0 { + return errors.New("timeout can not be negative") + } + c.WriteTimeout = timeout return nil } } // WithDB define func of configure redis db options -func WithDB(db int) RedisOption { - return func(o *RedisOptions) error { +func WithDB(db int) Option { + return func(c *clientConfig) error { if db < 0 { return errors.New("db can not be negative") } - o.redisOptions.DB = db + c.DB = db return nil } } // WithPoolSize define func of configure pool size options -func WithPoolSize(poolSize int) RedisOption { - return func(o *RedisOptions) error { +func WithPoolSize(poolSize int) Option { + return func(c *clientConfig) error { if poolSize <= 0 { return errors.New("pool size must be greater than 0") } - o.redisOptions.PoolSize = poolSize + c.PoolSize = poolSize return nil } }