feat: add component column recommendations for config tokens

- add component as a token6 config recommendation constant
- read component table column names during startup
- bind component config recommendations to component table columns
- add full and local autocomplete terms for component column paths

constants/recommend_keys.go

@@ -4,6 +4,8 @@ package constants
 const (
 	// DefaultScore define the default score for redissearch suggestion
 	DefaultScore = 1.0
+	// ComponentConfigKey define component config token used at token6
+	ComponentConfigKey = "component"
 )
 
 const (

database/query_component_columns.go

@@ -0,0 +1,28 @@
+// Package database define database operation functions
+package database
+
+import (
+	"context"
+
+	"modelRT/orm"
+
+	"gorm.io/gorm"
+)
+
+// QueryComponentColumnNames returns all column names from the component table.
+func QueryComponentColumnNames(ctx context.Context, db *gorm.DB) ([]string, error) {
+	columnTypes, err := db.WithContext(ctx).Migrator().ColumnTypes((&orm.Component{}).TableName())
+	if err != nil {
+		return nil, err
+	}
+
+	columnNames := make([]string, 0, len(columnTypes))
+	for _, columnType := range columnTypes {
+		columnName := columnType.Name()
+		if columnName == "" {
+			continue
+		}
+		columnNames = append(columnNames, columnName)
+	}
+	return columnNames, nil
+}

main.go

@@ -235,6 +235,18 @@ func main() {
 			panic(err)
 		}
 
+		componentColumnNames, err := database.QueryComponentColumnNames(ctx, tx)
+		if err != nil {
+			logger.Error(ctx, "query component table column names failed", "error", err)
+			panic(err)
+		}
+
+		err = model.StoreComponentColumnRecommend(ctx, fullParentPath, isLocalParentPath, componentColumnNames)
+		if err != nil {
+			logger.Error(ctx, "store component column recommend content failed", "error", err)
+			panic(err)
+		}
+
 		allMeasurement, err := database.GetAllMeasurements(ctx, tx)
 		if err != nil {
 			logger.Error(ctx, "load topologic info from postgres failed", "error", err)

model/component_column_recommend_model.go

@@ -0,0 +1,82 @@
+// Package model define model struct of model runtime service
+package model
+
+import (
+	"context"
+	"fmt"
+
+	"modelRT/constants"
+	"modelRT/diagram"
+
+	"github.com/RediSearch/redisearch-go/v2/redisearch"
+)
+
+// StoreComponentColumnRecommend binds token6 component config to component table column names.
+func StoreComponentColumnRecommend(ctx context.Context, compTagToFullPath map[string]string, isLocalCompTagToFullPath map[string]string, componentColumnNames []string) error {
+	rdb := diagram.GetRedisClientInstance()
+	pipe := rdb.Pipeline()
+
+	pipe.SAdd(ctx, constants.RedisAllConfigSetKey, constants.ComponentConfigKey)
+
+	if len(componentColumnNames) == 0 {
+		_, err := pipe.Exec(ctx)
+		return err
+	}
+
+	componentColumnMembers := stringSliceToAny(componentColumnNames)
+	pipe.SAdd(ctx, constants.RedisAllMeasTagSetKey, componentColumnMembers...)
+
+	sug := make([]redisearch.Suggestion, 0, len(compTagToFullPath)*len(componentColumnNames)*4)
+	for compTag, fullPath := range compTagToFullPath {
+		specCompMeasKey := fmt.Sprintf(constants.RedisSpecCompTagMeasSetKey, compTag)
+		pipe.SAdd(ctx, specCompMeasKey, componentColumnMembers...)
+
+		isLocalFullPath := isLocalCompTagToFullPath[compTag]
+		fullConfigTerm := fmt.Sprintf("%s.%s", fullPath, constants.ComponentConfigKey)
+		sug = append(sug, redisearch.Suggestion{
+			Term:  fullConfigTerm,
+			Score: constants.DefaultScore,
+		})
+
+		if isLocalFullPath != "" {
+			localConfigTerm := fmt.Sprintf("%s.%s", isLocalFullPath, constants.ComponentConfigKey)
+			sug = append(sug, redisearch.Suggestion{
+				Term:  localConfigTerm,
+				Score: constants.DefaultScore,
+			})
+		}
+
+		for _, columnName := range componentColumnNames {
+			fullColumnTerm := fmt.Sprintf("%s.%s.%s", fullPath, constants.ComponentConfigKey, columnName)
+			sug = append(sug, redisearch.Suggestion{
+				Term:  fullColumnTerm,
+				Score: constants.DefaultScore,
+			})
+
+			if isLocalFullPath == "" {
+				continue
+			}
+
+			localColumnTerm := fmt.Sprintf("%s.%s.%s", isLocalFullPath, constants.ComponentConfigKey, columnName)
+			sug = append(sug, redisearch.Suggestion{
+				Term:  localColumnTerm,
+				Score: constants.DefaultScore,
+			})
+		}
+	}
+
+	if len(sug) > 0 {
+		ac.AddTerms(sug...)
+	}
+
+	_, err := pipe.Exec(ctx)
+	return err
+}
+
+func stringSliceToAny(values []string) []any {
+	members := make([]any, 0, len(values))
+	for _, value := range values {
+		members = append(members, value)
+	}
+	return members
+}

model/redis_recommend.go

@@ -24,6 +24,7 @@ type SearchResult struct {
 	RecommendType constants.RecommendHierarchyType
 	QueryDatas    []string
 	IsFuzzy       bool
+	IsFallback    bool
 	Offset        int
 	Err           error
 }
@@ -96,11 +97,25 @@ func levelOneRedisSearch(ctx context.Context, rdb *redis.Client, hierarchy const
 		return
 	}
 
+	recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy)
+	if err != nil {
+		logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
+		fanInChan <- SearchResult{
+			RecommendType: hierarchy,
+			QueryDatas:    nil,
+			IsFuzzy:       true,
+			IsFallback:    true,
+			Err:           err,
+		}
+		return
+	}
+
 	fanInChan <- SearchResult{
 		RecommendType: hierarchy,
-		QueryDatas:    []string{},
+		QueryDatas:    recommends,
 		IsFuzzy:       true,
-		Offset:        offset,
+		IsFallback:    true,
+		Offset:        0,
 		Err:           nil,
 	}
 }
@@ -127,21 +142,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe
 			}
 
 			if result.RecommendType == constants.CompNSPathRecommendHierarchyType {
-				filterResults := make([]string, 0, len(result.QueryDatas))
+				result.QueryDatas = filterLocalNSPathResults(result.QueryDatas)
-				// TODO 增加 nspath 过滤
-				for _, queryData := range result.QueryDatas {
-					var nsPath string
-					if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
-						nsPath = queryData
-					} else {
-						nsPath = queryData[lastDotIndex+1:]
-					}
-
-					if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
-						filterResults = append(filterResults, queryData)
-					}
-				}
-				result.QueryDatas = filterResults
 			}
 			results[result.RecommendType.String()] = result
 		}
@@ -171,26 +172,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe
 			}
 
 			if result.RecommendType == constants.CompNSPathRecommendHierarchyType {
-				filterResults := make([]string, 0, len(result.QueryDatas))
+				result.QueryDatas = filterLocalNSPathResults(result.QueryDatas)
-				// TODO 增加 nspath 过滤
-				for _, queryData := range result.QueryDatas {
-					if queryData == "." {
-						filterResults = append(filterResults, queryData)
-						continue
-					}
-
-					var nsPath string
-					if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
-						nsPath = queryData
-					} else {
-						nsPath = queryData[lastDotIndex+1:]
-					}
-
-					if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
-						filterResults = append(filterResults, queryData)
-					}
-				}
-				result.QueryDatas = filterResults
 			}
 			results[result.RecommendType.String()] = result
 		}
@@ -302,6 +284,11 @@ func normalizeRecommendResults(input string, results map[string]SearchResult) ma
 		if result.Err != nil {
 			continue
 		}
+		if result.IsFallback {
+			result.Offset = maxRecommendLength(result.QueryDatas)
+			results[key] = result
+			continue
+		}
 		if !result.IsFuzzy {
 			result.Offset = calcSearchResultOffset(input, result.QueryDatas)
 		}
@@ -322,16 +309,27 @@ func calcSearchResultOffset(input string, recommends []string) int {
 	return minOffset
 }
 
+func maxRecommendLength(recommends []string) int {
+	var maxLen int
+	for _, recommend := range recommends {
+		if length := len([]rune(recommend)); length > maxLen {
+			maxLen = length
+		}
+	}
+	return maxLen
+}
+
 func trimRecommendTerms(recommends []string, offset int) []string {
 	resultRecommends := make([]string, 0, len(recommends))
 	seen := make(map[string]struct{})
 
 	for _, recommend := range recommends {
-		if offset > len(recommend) {
+		recommendRunes := []rune(recommend)
+		if offset > len(recommendRunes) {
 			continue
 		}
 
-		recommendTerm := recommend[offset:]
+		recommendTerm := string(recommendRunes[offset:])
 		if recommendTerm == "" {
 			continue
 		}
@@ -345,11 +343,68 @@ func trimRecommendTerms(recommends []string, offset int) []string {
 	return resultRecommends
 }
 
+func filterLocalNSPathResults(queryDatas []string) []string {
+	filterResults := make([]string, 0, len(queryDatas))
+	for _, queryData := range queryDatas {
+		if queryData == "." {
+			filterResults = append(filterResults, queryData)
+			continue
+		}
+
+		var nsPath string
+		if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
+			nsPath = queryData
+		} else {
+			nsPath = queryData[lastDotIndex+1:]
+		}
+
+		// TODO 考虑为 NSPathToIsLocalMap 增加读写锁或改为并发安全快照, 保证过滤数据一致性
+		if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
+			filterResults = append(filterResults, queryData)
+		}
+	}
+	return filterResults
+}
+
 func queryMemberFromSpecificsLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, keyPrefix string) ([]string, error) {
 	queryKey := getSpecificKeyByLength(hierarchy, keyPrefix)
 	return rdb.SMembers(ctx, queryKey).Result()
 }
 
+func fallbackAllCurrentLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType) ([]string, error) {
+	setKey, ok := getAllSetKeyByHierarchy(hierarchy)
+	if !ok {
+		return nil, fmt.Errorf("unsupported recommend hierarchy for fallback: %s", hierarchy)
+	}
+
+	members, err := rdb.SMembers(ctx, setKey).Result()
+	if err != nil && err != redis.Nil {
+		return nil, err
+	}
+	return members, nil
+}
+
+func getAllSetKeyByHierarchy(hierarchy constants.RecommendHierarchyType) (string, bool) {
+	switch hierarchy {
+	case constants.GridRecommendHierarchyType:
+		return constants.RedisAllGridSetKey, true
+	case constants.ZoneRecommendHierarchyType:
+		return constants.RedisAllZoneSetKey, true
+	case constants.StationRecommendHierarchyType:
+		return constants.RedisAllStationSetKey, true
+	case constants.CompNSPathRecommendHierarchyType:
+		return constants.RedisAllCompNSPathSetKey, true
+	case constants.CompTagRecommendHierarchyType:
+		return constants.RedisAllCompTagSetKey, true
+	case constants.ConfigRecommendHierarchyType:
+		return constants.RedisAllConfigSetKey, true
+	case constants.MeasTagRecommendHierarchyType:
+		return constants.RedisAllMeasTagSetKey, true
+	default:
+		return "", false
+	}
+}
+
 func getAllKeyByGridLevel(ctx context.Context, rdb *redis.Client, fanInChan chan SearchResult) {
 	setKey := constants.RedisAllGridSetKey
 	hierarchy := constants.GridRecommendHierarchyType
@@ -593,14 +648,30 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
 		return
 	}
 
+	isFallback := false
 	if len(recommends) == 0 {
 		logger.Info(ctx, "fuzzy search without result", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
+		recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy)
+		if err != nil {
+			logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
+			fanInChan <- SearchResult{
+				RecommendType: hierarchy,
+				QueryDatas:    nil,
+				IsFuzzy:       true,
+				IsFallback:    true,
+				Err:           err,
+			}
+			return
+		}
+		isFallback = true
+		offset = 0
 	}
 
 	fanInChan <- SearchResult{
 		RecommendType: hierarchy,
 		QueryDatas:    recommends,
 		IsFuzzy:       true,
+		IsFallback:    isFallback,
 		Offset:        offset,
 		Err:           nil,
 	}
@@ -610,7 +681,7 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
 func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, searchPrefix string, hierarchy constants.RecommendHierarchyType, recommendLenType string) ([]string, int, error) {
 	var configToken string
 	var comparePrefix string
-	searchInputLen := len(searchInput)
+	searchInputLen := len([]rune(searchInput))
 	compareHierarchyLen := int(hierarchy)
 	comparePrefix = searchPrefix
 	if hierarchy == constants.MeasTagRecommendHierarchyType {
@@ -641,10 +712,10 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
 		}
 
 		if len(results) == 0 {
-			// 如果没有结果，退一步（删除最后一个字节）并继续循环
+			// 如果没有结果，退一步（删除最后一个字符）并继续循环
-			// TODO 考虑使用其他方式代替 for 循环退一字节的查询方式
+			// TODO 考虑使用其他方式代替 for 循环退一字符的查询方式
-			searchInput = searchInput[:len(searchInput)-1]
+			searchInput = trimLastRune(searchInput)
-			searchInputLen = len(searchInput)
+			searchInputLen = len([]rune(searchInput))
 			continue
 		}
 
@@ -697,12 +768,20 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
 		}
 
 		if len(recommends) == 0 {
-			searchInput = searchInput[:len(searchInput)-1]
+			searchInput = trimLastRune(searchInput)
-			searchInputLen = len(searchInput)
+			searchInputLen = len([]rune(searchInput))
 			continue
 		}
 
-		return recommends, len(searchInput), nil
+		return recommends, len([]rune(searchInput)), nil
 	}
 	return []string{}, 0, nil
 }
+
+func trimLastRune(s string) string {
+	runes := []rune(s)
+	if len(runes) == 0 {
+		return s
+	}
+	return string(runes[:len(runes)-1])
+}

util/string.go

@@ -46,10 +46,12 @@ func GetLongestCommonPrefixLength(query string, result string) int {
 		return 0
 	}
 
-	minLen := min(len(query), len(result))
+	queryRunes := []rune(query)
+	resultRunes := []rune(result)
+	minLen := min(len(queryRunes), len(resultRunes))
 
 	for i := range minLen {
-		if query[i] != result[i] {
+		if queryRunes[i] != resultRunes[i] {
 			return i
 		}
 	}