From 42751c102067e4fe7ecafe59db1591eec237d508 Mon Sep 17 00:00:00 2001 From: douxu Date: Thu, 25 Dec 2025 17:17:20 +0800 Subject: [PATCH] optimize the logic for loading the cache of measurement nodes for traversing components --- constants/recommend_keys.go | 25 +-- database/query_component_measurement.go | 9 ++ .../measurement_injection.go | 10 +- main.go | 28 +++- model/attribute_group_model.go | 4 +- model/measurement_group_model.go | 152 +++++++++++++++--- 6 files changed, 184 insertions(+), 44 deletions(-) diff --git a/constants/recommend_keys.go b/constants/recommend_keys.go index 0f0c4a1..2fde23f 100644 --- a/constants/recommend_keys.go +++ b/constants/recommend_keys.go @@ -2,14 +2,19 @@ package constants const ( - // RedisAllGridSetKey define redis set key which store all grid keys - RedisAllGridSetKey = "grid_keys" + // DefaultScore define the default score for redissearch suggestion + DefaultScore = 1.0 +) - // RedisAllZoneSetKey define redis set key which store all zone keys - RedisAllZoneSetKey = "zone_keys" +const ( + // RedisAllGridSetKey define redis set key which store all grid tag keys + RedisAllGridSetKey = "grid_tag_keys" - // RedisAllStationSetKey define redis set key which store all station keys - RedisAllStationSetKey = "station_keys" + // RedisAllZoneSetKey define redis set key which store all zone tag keys + RedisAllZoneSetKey = "zone_tag_keys" + + // RedisAllStationSetKey define redis set key which store all station tag keys + RedisAllStationSetKey = "station_tag_keys" // RedisAllCompNSPathSetKey define redis set key which store all component nspath keys RedisAllCompNSPathSetKey = "component_nspath_keys" @@ -23,11 +28,11 @@ const ( // RedisAllMeasTagSetKey define redis set key which store all measurement tag keys RedisAllMeasTagSetKey = "measurement_tag_keys" - // RedisSpecGridZoneSetKey define redis set key which store all zone keys under specific grid - RedisSpecGridZoneSetKey = "%s_zone_keys" + // RedisSpecGridZoneSetKey define redis set key which store all zone tag keys under specific grid + RedisSpecGridZoneSetKey = "%s_zone_tag_keys" - // RedisSpecZoneStationSetKey define redis set key which store all station keys under specific zone - RedisSpecZoneStationSetKey = "%s_station_keys" + // RedisSpecZoneStationSetKey define redis set key which store all station tag keys under specific zone + RedisSpecZoneStationSetKey = "%s_station_tag_keys" // RedisSpecStationCompNSPATHSetKey define redis set key which store all component nspath keys under specific station RedisSpecStationCompNSPATHSetKey = "%s_component_nspath_keys" diff --git a/database/query_component_measurement.go b/database/query_component_measurement.go index fd327f4..44e8a97 100644 --- a/database/query_component_measurement.go +++ b/database/query_component_measurement.go @@ -26,6 +26,15 @@ func GetFullMeasurementSet(db *gorm.DB) (*orm.MeasurementSet, error) { CompTagToMeasTags: make(map[string][]string), } + var grids []orm.Grid + if err := db.Table("grid").Select("tagname").Scan(&grids).Error; err == nil { + for _, g := range grids { + if g.TAGNAME != "" { + mSet.AllGridTags = append(mSet.AllGridTags, g.TAGNAME) + } + } + } + var zones []struct { orm.Zone GridTag string `gorm:"column:grid_tag"` diff --git a/deploy/redis-test-data/measurments-recommend/measurement_injection.go b/deploy/redis-test-data/measurments-recommend/measurement_injection.go index 379e340..e444472 100644 --- a/deploy/redis-test-data/measurments-recommend/measurement_injection.go +++ b/deploy/redis-test-data/measurments-recommend/measurement_injection.go @@ -19,18 +19,18 @@ func init() { } const ( - gridKeysSet = "grid_keys" - zoneKeysSet = "zone_keys" - stationKeysSet = "station_keys" + gridKeysSet = "grid_tag_keys" + zoneKeysSet = "zone_tag_keys" + stationKeysSet = "station_tag_keys" componentNSPathKeysSet = "component_nspath_keys" componentTagKeysSet = "component_tag_keys" configKeysSet = "config_keys" measurementTagKeysSet = "measurement_tag_keys" // Grid -> Zone (e.g., grid1_zones_keys) - gridZoneSetKeyFormat = "grid%d_zone_keys" + gridZoneSetKeyFormat = "grid%d_zone_tag_keys" // Zone -> Station (e.g., zone1_1_stations_keys) - zoneStationSetKeyFormat = "zone%d_%d_station_keys" + zoneStationSetKeyFormat = "zone%d_%d_station_tag_keys" // Station -> NSPath (e.g., station1_1_1_components_nspath_keys) stationNSPathKeyFormat = "station%d_%d_%d_component_nspath_keys" // NSPath -> CompTag (e.g., ns1_1_1_1_components_tag_keys) diff --git a/main.go b/main.go index 415aec6..388fd1d 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,12 @@ import ( "flag" "fmt" "log" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "modelRT/alert" "modelRT/config" "modelRT/database" @@ -16,11 +22,6 @@ import ( "modelRT/pool" "modelRT/router" "modelRT/util" - "net/http" - "os" - "os/signal" - "path/filepath" - "syscall" locker "modelRT/distributedlock" _ "modelRT/docs" @@ -151,15 +152,26 @@ func main() { // panic(err) // } - compParamMap, err := database.GenAllAttributeMap(tx) + measurementSet, err := database.GetFullMeasurementSet(tx) + if err != nil { + logger.Error(ctx, "generate component measurement group failed", "error", err) + panic(err) + } + err = model.TraverseMeasurementGroupTables(ctx, *measurementSet) + if err != nil { + logger.Error(ctx, "store component measurement group into redis failed", "error", err) + panic(err) + } + + compAttrSet, err := database.GenAllAttributeMap(tx) if err != nil { logger.Error(ctx, "generate component attribute group failed", "error", err) panic(err) } - err = model.TraverseAttributeGroupTables(ctx, tx, compParamMap) + err = model.TraverseAttributeGroupTables(ctx, tx, compAttrSet) if err != nil { - logger.Error(ctx, "strore component attribute group into redis failed", "error", err) + logger.Error(ctx, "store component attribute group into redis failed", "error", err) panic(err) } diff --git a/model/attribute_group_model.go b/model/attribute_group_model.go index e56d833..81c9bf2 100644 --- a/model/attribute_group_model.go +++ b/model/attribute_group_model.go @@ -20,7 +20,7 @@ type columnParam struct { } // TraverseAttributeGroupTables define func to traverse component attribute group tables -func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compParamMap map[string]orm.AttributeSet) error { +func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compAttrSet map[string]orm.AttributeSet) error { var tableNames []string result := db.Model(&orm.ProjectManager{}).Pluck("name", &tableNames) @@ -79,7 +79,7 @@ func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compParamMap case "id": case "global_uuid": componentUUIDStr = value.(string) - attrSet, exists = compParamMap[componentUUIDStr] + attrSet, exists = compAttrSet[componentUUIDStr] if !exists { err := errors.New("can not find component tag from record by global uuid") logger.Error(ctx, "check component info by global uuid failed", "global_uuid", componentUUIDStr, "error", err) diff --git a/model/measurement_group_model.go b/model/measurement_group_model.go index a6f1cdb..fe1afd8 100644 --- a/model/measurement_group_model.go +++ b/model/measurement_group_model.go @@ -9,6 +9,9 @@ import ( "modelRT/diagram" "modelRT/logger" "modelRT/orm" + + "github.com/RediSearch/redisearch-go/v2/redisearch" + "github.com/redis/go-redis/v9" ) // TraverseMeasurementGroupTables define func to traverse component measurement group tables @@ -16,48 +19,159 @@ func TraverseMeasurementGroupTables(ctx context.Context, measSet orm.Measurement rdb := diagram.GetRedisClientInstance() pipe := rdb.Pipeline() - pipe.SAdd(ctx, constants.RedisAllGridSetKey, measSet.AllGridTags) - pipe.SAdd(ctx, constants.RedisAllZoneSetKey, measSet.AllZoneTags) - pipe.SAdd(ctx, constants.RedisAllStationSetKey, measSet.AllStationTags) - pipe.SAdd(ctx, constants.RedisAllGridSetKey, measSet.AllCompNSPaths) - pipe.SAdd(ctx, constants.RedisAllZoneSetKey, measSet.AllCompTags) - pipe.SAdd(ctx, constants.RedisAllConfigSetKey, "bay") - pipe.SAdd(ctx, constants.RedisAllStationSetKey, measSet.AllMeasTags) + zoneToGridPath := make(map[string]string) + for gridTag, zoneTags := range measSet.GridToZoneTags { + for _, zoneTag := range zoneTags { + zoneToGridPath[zoneTag] = gridTag + } + } + + stationToZonePath := make(map[string]string) + for zoneTag, stationTags := range measSet.ZoneToStationTags { + gridTag := zoneToGridPath[zoneTag] + for _, stationTag := range stationTags { + stationToZonePath[stationTag] = fmt.Sprintf("%s.%s", gridTag, zoneTag) + } + } + + compNSPathToStationPath := make(map[string]string) + for stationTag, nsPaths := range measSet.StationToCompNSPaths { + zonePath := stationToZonePath[stationTag] + for _, nsPath := range nsPaths { + compNSPathToStationPath[nsPath] = fmt.Sprintf("%s.%s", zonePath, stationTag) + } + } + + compTagToNSPathPath := make(map[string]string) + for nsPath, compTags := range measSet.CompNSPathToCompTags { + stationPath := compNSPathToStationPath[nsPath] + for _, compTag := range compTags { + compTagToNSPathPath[compTag] = fmt.Sprintf("%s.%s", stationPath, nsPath) + } + } + + // define a safe SAdd function to avoid errors caused by empty slices + safeSAdd := func(key string, members []string) { + if len(members) > 0 { + pipe.SAdd(ctx, key, members) + } + } + + safeSAdd(constants.RedisAllGridSetKey, measSet.AllGridTags) + gridSug := make([]redisearch.Suggestion, 0, len(measSet.AllGridTags)) + for _, gridTag := range measSet.AllGridTags { + gridSug = append(gridSug, redisearch.Suggestion{Term: gridTag, Score: constants.DefaultScore}) + } + ac.AddTerms(gridSug...) + + safeSAdd(constants.RedisAllZoneSetKey, measSet.AllZoneTags) + safeSAdd(constants.RedisAllStationSetKey, measSet.AllStationTags) + safeSAdd(constants.RedisAllCompNSPathSetKey, measSet.AllCompNSPaths) + safeSAdd(constants.RedisAllCompTagSetKey, measSet.AllCompTags) + safeSAdd(constants.RedisAllConfigSetKey, []string{"bay"}) + safeSAdd(constants.RedisAllMeasTagSetKey, measSet.AllMeasTags) // building the grid -> zones hierarchy for gridTag, zoneTags := range measSet.GridToZoneTags { - key := fmt.Sprintf(constants.RedisSpecGridZoneSetKey, gridTag) - pipe.SAdd(ctx, key, zoneTags) + sug := make([]redisearch.Suggestion, 0, len(zoneTags)) + for _, zoneTag := range zoneTags { + term := fmt.Sprintf("%s.%s", gridTag, zoneTag) + sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore}) + } + safeSAdd(fmt.Sprintf(constants.RedisSpecGridZoneSetKey, gridTag), zoneTags) + ac.AddTerms(sug...) } // building the zone -> stations hierarchy for zoneTag, stationTags := range measSet.ZoneToStationTags { - key := fmt.Sprintf(constants.RedisSpecZoneStationSetKey, zoneTag) - pipe.SAdd(ctx, key, stationTags) + sug := make([]redisearch.Suggestion, 0, len(stationTags)) + gridTag, exists := zoneToGridPath[zoneTag] + if !exists { + err := fmt.Errorf("zone tag to grid tag mapping not found for zoneTag: %s", zoneTag) + logger.Error(ctx, "zone tag to grid tag mapping not found", "zoneTag", zoneTag, "error", err) + return err + } + + for _, stationTag := range stationTags { + term := fmt.Sprintf("%s.%s.%s", gridTag, zoneTag, stationTag) + sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore}) + } + + safeSAdd(fmt.Sprintf(constants.RedisSpecZoneStationSetKey, zoneTag), stationTags) + ac.AddTerms(sug...) } // building the station -> component nspaths hierarchy for stationTag, compNSPaths := range measSet.StationToCompNSPaths { - key := fmt.Sprintf(constants.RedisSpecStationCompNSPATHSetKey, stationTag) - pipe.SAdd(ctx, key, compNSPaths) + sug := make([]redisearch.Suggestion, 0, len(compNSPaths)) + parentPath, exists := stationToZonePath[stationTag] + if !exists { + err := fmt.Errorf("station tag to zone tag mapping not found for stationTag: %s", stationTag) + logger.Error(ctx, "zone tag to grid tag mapping not found", "stationTag", stationTag, "error", err) + return err + } + + for _, nsPath := range compNSPaths { + term := fmt.Sprintf("%s.%s.%s", parentPath, stationTag, nsPath) + sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore}) + } + safeSAdd(fmt.Sprintf(constants.RedisSpecStationCompNSPATHSetKey, stationTag), compNSPaths) + ac.AddTerms(sug...) } // building the component nspath -> component tags hierarchy for compNSPath, compTags := range measSet.CompNSPathToCompTags { - key := fmt.Sprintf(constants.RedisSpecCompNSPathCompTagSetKey, compNSPath) - pipe.SAdd(ctx, key, compTags) + sug := make([]redisearch.Suggestion, 0, len(compTags)) + parentPath, exists := compNSPathToStationPath[compNSPath] + if !exists { + err := fmt.Errorf("component nspath tag to station tag mapping not found for compNSPath: %s", compNSPath) + logger.Error(ctx, "component nspath tag to station tag mapping not found", "compNSPath", compNSPath, "error", err) + return err + } + + for _, compTag := range compTags { + term := fmt.Sprintf("%s.%s.%s", parentPath, compNSPath, compTag) + sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore}) + } + safeSAdd(fmt.Sprintf(constants.RedisSpecCompNSPathCompTagSetKey, compNSPath), compTags) + ac.AddTerms(sug...) } // building the component tag -> measurement tags hierarchy for compTag, measTags := range measSet.CompTagToMeasTags { - key := fmt.Sprintf(constants.RedisSpecCompTagMeasSetKey, compTag) - pipe.SAdd(ctx, key, measTags) + sug := make([]redisearch.Suggestion, 0, len(measTags)) + parentPath, exists := compTagToNSPathPath[compTag] + if !exists { + err := fmt.Errorf("component tag to component nspath mapping not found for compTag: %s", compTag) + logger.Error(ctx, "component tag to component nspath mapping not found", "compTag", compTag, "error", err) + return err + } + + for _, measTag := range measTags { + term := fmt.Sprintf("%s.%s.%s", parentPath, compTag, measTag) + sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore}) + } + safeSAdd(fmt.Sprintf(constants.RedisSpecCompTagMeasSetKey, compTag), measTags) + ac.AddTerms(sug...) } - _, err := pipe.Exec(ctx) + cmders, err := pipe.Exec(ctx) if err != nil { - logger.Error(ctx, "init component measurement group recommend content failed", "error", err) + logger.Error(ctx, "pipeline execution failed", "error", err) return err } + + // check for errors in each subcommand of the pipeline + for _, cmder := range cmders { + cmdErr := cmder.Err() + if cmdErr != nil { + logger.Error(ctx, "individual redis command failed", + "command", cmder.Name(), + "args", cmder.Args(), + "error", cmdErr) + } + return cmdErr + } + return nil }