fix: improve measurement recommend fallback and rune offsets

- fallback to all current-level candidates when fuzzy search has no result
- track fallback results and calculate fallback offsets from max rune length
- use rune-based prefix offsets and trimming for non-ASCII tokens
- reuse local nspath filtering for component nspath recommendations

model/redis_recommend.go

@@ -24,6 +24,7 @@ type SearchResult struct {
 	RecommendType constants.RecommendHierarchyType
 	QueryDatas    []string
 	IsFuzzy       bool
+	IsFallback    bool
 	Offset        int
 	Err           error
 }
@@ -96,11 +97,25 @@ func levelOneRedisSearch(ctx context.Context, rdb *redis.Client, hierarchy const
 		return
 	}
 
+	recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy)
+	if err != nil {
+		logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
+		fanInChan <- SearchResult{
+			RecommendType: hierarchy,
+			QueryDatas:    nil,
+			IsFuzzy:       true,
+			IsFallback:    true,
+			Err:           err,
+		}
+		return
+	}
+
 	fanInChan <- SearchResult{
 		RecommendType: hierarchy,
-		QueryDatas:    []string{},
+		QueryDatas:    recommends,
 		IsFuzzy:       true,
-		Offset:        offset,
+		IsFallback:    true,
+		Offset:        0,
 		Err:           nil,
 	}
 }
@@ -127,21 +142,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe
 			}
 
 			if result.RecommendType == constants.CompNSPathRecommendHierarchyType {
-				filterResults := make([]string, 0, len(result.QueryDatas))
+				result.QueryDatas = filterLocalNSPathResults(result.QueryDatas)
-				// TODO 增加 nspath 过滤
-				for _, queryData := range result.QueryDatas {
-					var nsPath string
-					if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
-						nsPath = queryData
-					} else {
-						nsPath = queryData[lastDotIndex+1:]
-					}
-
-					if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
-						filterResults = append(filterResults, queryData)
-					}
-				}
-				result.QueryDatas = filterResults
 			}
 			results[result.RecommendType.String()] = result
 		}
@@ -171,26 +172,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe
 			}
 
 			if result.RecommendType == constants.CompNSPathRecommendHierarchyType {
-				filterResults := make([]string, 0, len(result.QueryDatas))
+				result.QueryDatas = filterLocalNSPathResults(result.QueryDatas)
-				// TODO 增加 nspath 过滤
-				for _, queryData := range result.QueryDatas {
-					if queryData == "." {
-						filterResults = append(filterResults, queryData)
-						continue
-					}
-
-					var nsPath string
-					if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
-						nsPath = queryData
-					} else {
-						nsPath = queryData[lastDotIndex+1:]
-					}
-
-					if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
-						filterResults = append(filterResults, queryData)
-					}
-				}
-				result.QueryDatas = filterResults
 			}
 			results[result.RecommendType.String()] = result
 		}
@@ -302,6 +284,11 @@ func normalizeRecommendResults(input string, results map[string]SearchResult) ma
 		if result.Err != nil {
 			continue
 		}
+		if result.IsFallback {
+			result.Offset = maxRecommendLength(result.QueryDatas)
+			results[key] = result
+			continue
+		}
 		if !result.IsFuzzy {
 			result.Offset = calcSearchResultOffset(input, result.QueryDatas)
 		}
@@ -322,16 +309,27 @@ func calcSearchResultOffset(input string, recommends []string) int {
 	return minOffset
 }
 
+func maxRecommendLength(recommends []string) int {
+	var maxLen int
+	for _, recommend := range recommends {
+		if length := len([]rune(recommend)); length > maxLen {
+			maxLen = length
+		}
+	}
+	return maxLen
+}
+
 func trimRecommendTerms(recommends []string, offset int) []string {
 	resultRecommends := make([]string, 0, len(recommends))
 	seen := make(map[string]struct{})
 
 	for _, recommend := range recommends {
-		if offset > len(recommend) {
+		recommendRunes := []rune(recommend)
+		if offset > len(recommendRunes) {
 			continue
 		}
 
-		recommendTerm := recommend[offset:]
+		recommendTerm := string(recommendRunes[offset:])
 		if recommendTerm == "" {
 			continue
 		}
@@ -345,11 +343,68 @@ func trimRecommendTerms(recommends []string, offset int) []string {
 	return resultRecommends
 }
 
+func filterLocalNSPathResults(queryDatas []string) []string {
+	filterResults := make([]string, 0, len(queryDatas))
+	for _, queryData := range queryDatas {
+		if queryData == "." {
+			filterResults = append(filterResults, queryData)
+			continue
+		}
+
+		var nsPath string
+		if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
+			nsPath = queryData
+		} else {
+			nsPath = queryData[lastDotIndex+1:]
+		}
+
+		// TODO 考虑为 NSPathToIsLocalMap 增加读写锁或改为并发安全快照, 保证过滤数据一致性
+		if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
+			filterResults = append(filterResults, queryData)
+		}
+	}
+	return filterResults
+}
+
 func queryMemberFromSpecificsLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, keyPrefix string) ([]string, error) {
 	queryKey := getSpecificKeyByLength(hierarchy, keyPrefix)
 	return rdb.SMembers(ctx, queryKey).Result()
 }
 
+func fallbackAllCurrentLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType) ([]string, error) {
+	setKey, ok := getAllSetKeyByHierarchy(hierarchy)
+	if !ok {
+		return nil, fmt.Errorf("unsupported recommend hierarchy for fallback: %s", hierarchy)
+	}
+
+	members, err := rdb.SMembers(ctx, setKey).Result()
+	if err != nil && err != redis.Nil {
+		return nil, err
+	}
+	return members, nil
+}
+
+func getAllSetKeyByHierarchy(hierarchy constants.RecommendHierarchyType) (string, bool) {
+	switch hierarchy {
+	case constants.GridRecommendHierarchyType:
+		return constants.RedisAllGridSetKey, true
+	case constants.ZoneRecommendHierarchyType:
+		return constants.RedisAllZoneSetKey, true
+	case constants.StationRecommendHierarchyType:
+		return constants.RedisAllStationSetKey, true
+	case constants.CompNSPathRecommendHierarchyType:
+		return constants.RedisAllCompNSPathSetKey, true
+	case constants.CompTagRecommendHierarchyType:
+		return constants.RedisAllCompTagSetKey, true
+	case constants.ConfigRecommendHierarchyType:
+		return constants.RedisAllConfigSetKey, true
+	case constants.MeasTagRecommendHierarchyType:
+		return constants.RedisAllMeasTagSetKey, true
+	default:
+		return "", false
+	}
+}
+
 func getAllKeyByGridLevel(ctx context.Context, rdb *redis.Client, fanInChan chan SearchResult) {
 	setKey := constants.RedisAllGridSetKey
 	hierarchy := constants.GridRecommendHierarchyType
@@ -593,14 +648,30 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
 		return
 	}
 
+	isFallback := false
 	if len(recommends) == 0 {
 		logger.Info(ctx, "fuzzy search without result", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
+		recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy)
+		if err != nil {
+			logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
+			fanInChan <- SearchResult{
+				RecommendType: hierarchy,
+				QueryDatas:    nil,
+				IsFuzzy:       true,
+				IsFallback:    true,
+				Err:           err,
+			}
+			return
+		}
+		isFallback = true
+		offset = 0
 	}
 
 	fanInChan <- SearchResult{
 		RecommendType: hierarchy,
 		QueryDatas:    recommends,
 		IsFuzzy:       true,
+		IsFallback:    isFallback,
 		Offset:        offset,
 		Err:           nil,
 	}
@@ -610,7 +681,7 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
 func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, searchPrefix string, hierarchy constants.RecommendHierarchyType, recommendLenType string) ([]string, int, error) {
 	var configToken string
 	var comparePrefix string
-	searchInputLen := len(searchInput)
+	searchInputLen := len([]rune(searchInput))
 	compareHierarchyLen := int(hierarchy)
 	comparePrefix = searchPrefix
 	if hierarchy == constants.MeasTagRecommendHierarchyType {
@@ -641,10 +712,10 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
 		}
 
 		if len(results) == 0 {
-			// 如果没有结果，退一步（删除最后一个字节）并继续循环
+			// 如果没有结果，退一步（删除最后一个字符）并继续循环
-			// TODO 考虑使用其他方式代替 for 循环退一字节的查询方式
+			// TODO 考虑使用其他方式代替 for 循环退一字符的查询方式
-			searchInput = searchInput[:len(searchInput)-1]
+			searchInput = trimLastRune(searchInput)
-			searchInputLen = len(searchInput)
+			searchInputLen = len([]rune(searchInput))
 			continue
 		}
 
@@ -697,12 +768,20 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
 		}
 
 		if len(recommends) == 0 {
-			searchInput = searchInput[:len(searchInput)-1]
+			searchInput = trimLastRune(searchInput)
-			searchInputLen = len(searchInput)
+			searchInputLen = len([]rune(searchInput))
 			continue
 		}
 
-		return recommends, len(searchInput), nil
+		return recommends, len([]rune(searchInput)), nil
 	}
 	return []string{}, 0, nil
 }
+
+func trimLastRune(s string) string {
+	runes := []rune(s)
+	if len(runes) == 0 {
+		return s
+	}
+	return string(runes[:len(runes)-1])
+}

util/string.go

@@ -46,10 +46,12 @@ func GetLongestCommonPrefixLength(query string, result string) int {
 		return 0
 	}
 
-	minLen := min(len(query), len(result))
+	queryRunes := []rune(query)
+	resultRunes := []rune(result)
+	minLen := min(len(queryRunes), len(resultRunes))
 
 	for i := range minLen {
-		if query[i] != result[i] {
+		if queryRunes[i] != resultRunes[i] {
 			return i
 		}
 	}