optimize the logic for loading the cache of measurement nodes for traversing components

constants/recommend_keys.go

@@ -2,14 +2,19 @@
 package constants
 
 const (
-	// RedisAllGridSetKey define redis set key which store all grid keys
-	RedisAllGridSetKey = "grid_keys"
+	// DefaultScore define the default score for redissearch suggestion
+	DefaultScore = 1.0
+)
 
-	// RedisAllZoneSetKey define redis set key which store all zone keys
-	RedisAllZoneSetKey = "zone_keys"
+const (
+	// RedisAllGridSetKey define redis set key which store all grid tag keys
+	RedisAllGridSetKey = "grid_tag_keys"
 
-	// RedisAllStationSetKey define redis set key which store all station keys
-	RedisAllStationSetKey = "station_keys"
+	// RedisAllZoneSetKey define redis set key which store all zone tag keys
+	RedisAllZoneSetKey = "zone_tag_keys"
+
+	// RedisAllStationSetKey define redis set key which store all station tag keys
+	RedisAllStationSetKey = "station_tag_keys"
 
 	// RedisAllCompNSPathSetKey define redis set key which store all component nspath keys
 	RedisAllCompNSPathSetKey = "component_nspath_keys"
@@ -23,11 +28,11 @@ const (
 	// RedisAllMeasTagSetKey define redis set key which store all measurement tag keys
 	RedisAllMeasTagSetKey = "measurement_tag_keys"
 
-	// RedisSpecGridZoneSetKey define redis set key which store all zone keys under specific grid
-	RedisSpecGridZoneSetKey = "%s_zone_keys"
+	// RedisSpecGridZoneSetKey define redis set key which store all zone tag keys under specific grid
+	RedisSpecGridZoneSetKey = "%s_zone_tag_keys"
 
-	// RedisSpecZoneStationSetKey define redis set key which store all station keys under specific zone
-	RedisSpecZoneStationSetKey = "%s_station_keys"
+	// RedisSpecZoneStationSetKey define redis set key which store all station tag keys under specific zone
+	RedisSpecZoneStationSetKey = "%s_station_tag_keys"
 
 	// RedisSpecStationCompNSPATHSetKey define redis set key which store all component nspath keys under specific station
 	RedisSpecStationCompNSPATHSetKey = "%s_component_nspath_keys"

database/query_component_measurement.go

@@ -26,6 +26,15 @@ func GetFullMeasurementSet(db *gorm.DB) (*orm.MeasurementSet, error) {
 		CompTagToMeasTags:    make(map[string][]string),
 	}
 
+	var grids []orm.Grid
+	if err := db.Table("grid").Select("tagname").Scan(&grids).Error; err == nil {
+		for _, g := range grids {
+			if g.TAGNAME != "" {
+				mSet.AllGridTags = append(mSet.AllGridTags, g.TAGNAME)
+			}
+		}
+	}
+
 	var zones []struct {
 		orm.Zone
 		GridTag string `gorm:"column:grid_tag"`

deploy/redis-test-data/measurments-recommend/measurement_injection.go

@@ -19,18 +19,18 @@ func init() {
 }
 
 const (
-	gridKeysSet            = "grid_keys"
-	zoneKeysSet            = "zone_keys"
-	stationKeysSet         = "station_keys"
+	gridKeysSet            = "grid_tag_keys"
+	zoneKeysSet            = "zone_tag_keys"
+	stationKeysSet         = "station_tag_keys"
 	componentNSPathKeysSet = "component_nspath_keys"
 	componentTagKeysSet    = "component_tag_keys"
 	configKeysSet          = "config_keys"
 	measurementTagKeysSet  = "measurement_tag_keys"
 
 	// Grid -> Zone (e.g., grid1_zones_keys)
-	gridZoneSetKeyFormat = "grid%d_zone_keys"
+	gridZoneSetKeyFormat = "grid%d_zone_tag_keys"
 	// Zone -> Station (e.g., zone1_1_stations_keys)
-	zoneStationSetKeyFormat = "zone%d_%d_station_keys"
+	zoneStationSetKeyFormat = "zone%d_%d_station_tag_keys"
 	// Station -> NSPath (e.g., station1_1_1_components_nspath_keys)
 	stationNSPathKeyFormat = "station%d_%d_%d_component_nspath_keys"
 	// NSPath -> CompTag (e.g., ns1_1_1_1_components_tag_keys)

main.go

@@ -7,6 +7,12 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
 	"modelRT/alert"
 	"modelRT/config"
 	"modelRT/database"
@@ -16,11 +22,6 @@ import (
 	"modelRT/pool"
 	"modelRT/router"
 	"modelRT/util"
-	"net/http"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
 
 	locker "modelRT/distributedlock"
 	_ "modelRT/docs"
@@ -151,15 +152,26 @@ func main() {
 		// 	panic(err)
 		// }
 
-		compParamMap, err := database.GenAllAttributeMap(tx)
+		measurementSet, err := database.GetFullMeasurementSet(tx)
+		if err != nil {
+			logger.Error(ctx, "generate component measurement group failed", "error", err)
+			panic(err)
+		}
+		err = model.TraverseMeasurementGroupTables(ctx, *measurementSet)
+		if err != nil {
+			logger.Error(ctx, "store component measurement group into redis failed", "error", err)
+			panic(err)
+		}
+
+		compAttrSet, err := database.GenAllAttributeMap(tx)
 		if err != nil {
 			logger.Error(ctx, "generate component attribute group failed", "error", err)
 			panic(err)
 		}
 
-		err = model.TraverseAttributeGroupTables(ctx, tx, compParamMap)
+		err = model.TraverseAttributeGroupTables(ctx, tx, compAttrSet)
 		if err != nil {
-			logger.Error(ctx, "strore component attribute group into redis failed", "error", err)
+			logger.Error(ctx, "store component attribute group into redis failed", "error", err)
 			panic(err)
 		}
 

model/attribute_group_model.go

@@ -20,7 +20,7 @@ type columnParam struct {
 }
 
 // TraverseAttributeGroupTables define func to traverse component attribute group tables
-func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compParamMap map[string]orm.AttributeSet) error {
+func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compAttrSet map[string]orm.AttributeSet) error {
 	var tableNames []string
 
 	result := db.Model(&orm.ProjectManager{}).Pluck("name", &tableNames)
@@ -79,7 +79,7 @@ func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compParamMap
 				case "id":
 				case "global_uuid":
 					componentUUIDStr = value.(string)
-					attrSet, exists = compParamMap[componentUUIDStr]
+					attrSet, exists = compAttrSet[componentUUIDStr]
 					if !exists {
 						err := errors.New("can not find component tag from record by global uuid")
 						logger.Error(ctx, "check component info by global uuid failed", "global_uuid", componentUUIDStr, "error", err)

model/measurement_group_model.go

@@ -9,6 +9,9 @@ import (
 	"modelRT/diagram"
 	"modelRT/logger"
 	"modelRT/orm"
+
+	"github.com/RediSearch/redisearch-go/v2/redisearch"
+	"github.com/redis/go-redis/v9"
 )
 
 // TraverseMeasurementGroupTables define func to traverse component measurement group tables
@@ -16,48 +19,159 @@ func TraverseMeasurementGroupTables(ctx context.Context, measSet orm.Measurement
 	rdb := diagram.GetRedisClientInstance()
 	pipe := rdb.Pipeline()
 
-	pipe.SAdd(ctx, constants.RedisAllGridSetKey, measSet.AllGridTags)
-	pipe.SAdd(ctx, constants.RedisAllZoneSetKey, measSet.AllZoneTags)
-	pipe.SAdd(ctx, constants.RedisAllStationSetKey, measSet.AllStationTags)
-	pipe.SAdd(ctx, constants.RedisAllGridSetKey, measSet.AllCompNSPaths)
-	pipe.SAdd(ctx, constants.RedisAllZoneSetKey, measSet.AllCompTags)
-	pipe.SAdd(ctx, constants.RedisAllConfigSetKey, "bay")
-	pipe.SAdd(ctx, constants.RedisAllStationSetKey, measSet.AllMeasTags)
+	zoneToGridPath := make(map[string]string)
+	for gridTag, zoneTags := range measSet.GridToZoneTags {
+		for _, zoneTag := range zoneTags {
+			zoneToGridPath[zoneTag] = gridTag
+		}
+	}
+
+	stationToZonePath := make(map[string]string)
+	for zoneTag, stationTags := range measSet.ZoneToStationTags {
+		gridTag := zoneToGridPath[zoneTag]
+		for _, stationTag := range stationTags {
+			stationToZonePath[stationTag] = fmt.Sprintf("%s.%s", gridTag, zoneTag)
+		}
+	}
+
+	compNSPathToStationPath := make(map[string]string)
+	for stationTag, nsPaths := range measSet.StationToCompNSPaths {
+		zonePath := stationToZonePath[stationTag]
+		for _, nsPath := range nsPaths {
+			compNSPathToStationPath[nsPath] = fmt.Sprintf("%s.%s", zonePath, stationTag)
+		}
+	}
+
+	compTagToNSPathPath := make(map[string]string)
+	for nsPath, compTags := range measSet.CompNSPathToCompTags {
+		stationPath := compNSPathToStationPath[nsPath]
+		for _, compTag := range compTags {
+			compTagToNSPathPath[compTag] = fmt.Sprintf("%s.%s", stationPath, nsPath)
+		}
+	}
+
+	// define a safe SAdd function to avoid errors caused by empty slices
+	safeSAdd := func(key string, members []string) {
+		if len(members) > 0 {
+			pipe.SAdd(ctx, key, members)
+		}
+	}
+
+	safeSAdd(constants.RedisAllGridSetKey, measSet.AllGridTags)
+	gridSug := make([]redisearch.Suggestion, 0, len(measSet.AllGridTags))
+	for _, gridTag := range measSet.AllGridTags {
+		gridSug = append(gridSug, redisearch.Suggestion{Term: gridTag, Score: constants.DefaultScore})
+	}
+	ac.AddTerms(gridSug...)
+
+	safeSAdd(constants.RedisAllZoneSetKey, measSet.AllZoneTags)
+	safeSAdd(constants.RedisAllStationSetKey, measSet.AllStationTags)
+	safeSAdd(constants.RedisAllCompNSPathSetKey, measSet.AllCompNSPaths)
+	safeSAdd(constants.RedisAllCompTagSetKey, measSet.AllCompTags)
+	safeSAdd(constants.RedisAllConfigSetKey, []string{"bay"})
+	safeSAdd(constants.RedisAllMeasTagSetKey, measSet.AllMeasTags)
 
 	// building the grid -> zones hierarchy
 	for gridTag, zoneTags := range measSet.GridToZoneTags {
-		key := fmt.Sprintf(constants.RedisSpecGridZoneSetKey, gridTag)
-		pipe.SAdd(ctx, key, zoneTags)
+		sug := make([]redisearch.Suggestion, 0, len(zoneTags))
+		for _, zoneTag := range zoneTags {
+			term := fmt.Sprintf("%s.%s", gridTag, zoneTag)
+			sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore})
+		}
+		safeSAdd(fmt.Sprintf(constants.RedisSpecGridZoneSetKey, gridTag), zoneTags)
+		ac.AddTerms(sug...)
 	}
 
 	// building the zone -> stations hierarchy
 	for zoneTag, stationTags := range measSet.ZoneToStationTags {
-		key := fmt.Sprintf(constants.RedisSpecZoneStationSetKey, zoneTag)
-		pipe.SAdd(ctx, key, stationTags)
+		sug := make([]redisearch.Suggestion, 0, len(stationTags))
+		gridTag, exists := zoneToGridPath[zoneTag]
+		if !exists {
+			err := fmt.Errorf("zone tag to grid tag mapping not found for zoneTag: %s", zoneTag)
+			logger.Error(ctx, "zone tag to grid tag mapping not found", "zoneTag", zoneTag, "error", err)
+			return err
+		}
+
+		for _, stationTag := range stationTags {
+			term := fmt.Sprintf("%s.%s.%s", gridTag, zoneTag, stationTag)
+			sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore})
+		}
+
+		safeSAdd(fmt.Sprintf(constants.RedisSpecZoneStationSetKey, zoneTag), stationTags)
+		ac.AddTerms(sug...)
 	}
 
 	// building the station -> component nspaths hierarchy
 	for stationTag, compNSPaths := range measSet.StationToCompNSPaths {
-		key := fmt.Sprintf(constants.RedisSpecStationCompNSPATHSetKey, stationTag)
-		pipe.SAdd(ctx, key, compNSPaths)
+		sug := make([]redisearch.Suggestion, 0, len(compNSPaths))
+		parentPath, exists := stationToZonePath[stationTag]
+		if !exists {
+			err := fmt.Errorf("station tag to zone tag mapping not found for stationTag: %s", stationTag)
+			logger.Error(ctx, "zone tag to grid tag mapping not found", "stationTag", stationTag, "error", err)
+			return err
+		}
+
+		for _, nsPath := range compNSPaths {
+			term := fmt.Sprintf("%s.%s.%s", parentPath, stationTag, nsPath)
+			sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore})
+		}
+		safeSAdd(fmt.Sprintf(constants.RedisSpecStationCompNSPATHSetKey, stationTag), compNSPaths)
+		ac.AddTerms(sug...)
 	}
 
 	// building the component nspath -> component tags hierarchy
 	for compNSPath, compTags := range measSet.CompNSPathToCompTags {
-		key := fmt.Sprintf(constants.RedisSpecCompNSPathCompTagSetKey, compNSPath)
-		pipe.SAdd(ctx, key, compTags)
+		sug := make([]redisearch.Suggestion, 0, len(compTags))
+		parentPath, exists := compNSPathToStationPath[compNSPath]
+		if !exists {
+			err := fmt.Errorf("component nspath tag to station tag mapping not found for compNSPath: %s", compNSPath)
+			logger.Error(ctx, "component nspath tag to station tag mapping not found", "compNSPath", compNSPath, "error", err)
+			return err
+		}
+
+		for _, compTag := range compTags {
+			term := fmt.Sprintf("%s.%s.%s", parentPath, compNSPath, compTag)
+			sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore})
+		}
+		safeSAdd(fmt.Sprintf(constants.RedisSpecCompNSPathCompTagSetKey, compNSPath), compTags)
+		ac.AddTerms(sug...)
 	}
 
 	// building the component tag -> measurement tags hierarchy
 	for compTag, measTags := range measSet.CompTagToMeasTags {
-		key := fmt.Sprintf(constants.RedisSpecCompTagMeasSetKey, compTag)
-		pipe.SAdd(ctx, key, measTags)
+		sug := make([]redisearch.Suggestion, 0, len(measTags))
+		parentPath, exists := compTagToNSPathPath[compTag]
+		if !exists {
+			err := fmt.Errorf("component tag to component nspath mapping not found for compTag: %s", compTag)
+			logger.Error(ctx, "component tag to component nspath mapping not found", "compTag", compTag, "error", err)
+			return err
+		}
+
+		for _, measTag := range measTags {
+			term := fmt.Sprintf("%s.%s.%s", parentPath, compTag, measTag)
+			sug = append(sug, redisearch.Suggestion{Term: term, Score: constants.DefaultScore})
+		}
+		safeSAdd(fmt.Sprintf(constants.RedisSpecCompTagMeasSetKey, compTag), measTags)
+		ac.AddTerms(sug...)
 	}
 
-	_, err := pipe.Exec(ctx)
+	cmders, err := pipe.Exec(ctx)
 	if err != nil {
-		logger.Error(ctx, "init component measurement group recommend content failed", "error", err)
+		logger.Error(ctx, "pipeline execution failed", "error", err)
 		return err
 	}
+
+	// check for errors in each subcommand of the pipeline
+	for _, cmder := range cmders {
+		cmdErr := cmder.Err()
+		if cmdErr != nil {
+			logger.Error(ctx, "individual redis command failed",
+				"command", cmder.Name(),
+				"args", cmder.Args(),
+				"error", cmdErr)
+		}
+		return cmdErr
+	}
+
 	return nil
 }