From b53746efcd181daad60c2956624b52adb1b2f9c1 Mon Sep 17 00:00:00 2001 From: douxu Date: Tue, 7 Jul 2026 17:11:25 +0800 Subject: [PATCH] fix: improve redis recommendation fuzzy matching - add bounded errgroup handling for attribute group suggestion initialization - add config-level suggestions and propagate Redis/RediSearch write errors - constrain fuzzy recommendation candidates by hierarchy-specific Redis sets - avoid broad fallback results when concrete matches exist - preserve full matched prefix when calculating fuzzy recommendation offset - add tests for fuzzy recommendation offset calculation --- model/attribute_group_recommend_model.go | 75 +++++++++--- model/redis_recommend.go | 150 +++++++++++++++++++++-- model/redis_recommend_test.go | 40 ++++++ 3 files changed, 236 insertions(+), 29 deletions(-) create mode 100644 model/redis_recommend_test.go diff --git a/model/attribute_group_recommend_model.go b/model/attribute_group_recommend_model.go index 2db4baa..22253df 100644 --- a/model/attribute_group_recommend_model.go +++ b/model/attribute_group_recommend_model.go @@ -12,6 +12,7 @@ import ( "modelRT/orm" "github.com/RediSearch/redisearch-go/v2/redisearch" + "golang.org/x/sync/errgroup" "gorm.io/gorm" ) @@ -20,6 +21,13 @@ type columnParam struct { AttributeGroup map[string]any } +type attributeGroupRecommendJob struct { + AttributeSet orm.AttributeSet + FullPath string + IsLocalFullPath string + ColumnParam columnParam +} + // TraverseAttributeGroupTables define func to traverse component attribute group tables func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compTagToFullPath map[string]string, isLocalCompTagToFullPath map[string]string, compAttrSet map[string]orm.AttributeSet) error { var tableNames []string @@ -38,6 +46,7 @@ func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compTagToFul return nil } + jobs := make([]attributeGroupRecommendJob, 0) for _, tableName := range tableNames { var records []map[string]any err := db.Table(tableName).Find(&records).Error @@ -102,13 +111,27 @@ func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compTagToFul AttributeType: attributeType, AttributeGroup: attributeGroup, } - go storeAttributeGroup(ctx, attrSet, fullPath, isLocalfullPath, columnParam) + jobs = append(jobs, attributeGroupRecommendJob{ + AttributeSet: attrSet, + FullPath: fullPath, + IsLocalFullPath: isLocalfullPath, + ColumnParam: columnParam, + }) } } - return nil + + group, groupCtx := errgroup.WithContext(ctx) + group.SetLimit(16) + for _, job := range jobs { + job := job + group.Go(func() error { + return storeAttributeGroup(groupCtx, job.AttributeSet, job.FullPath, job.IsLocalFullPath, job.ColumnParam) + }) + } + return group.Wait() } -func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, fullPath string, isLocalFullPath string, colParams columnParam) { +func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, fullPath string, isLocalFullPath string, colParams columnParam) error { rdb := diagram.GetRedisClientInstance() pipe := rdb.Pipeline() @@ -121,18 +144,25 @@ func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, ful attrbutesGroups := make([]any, 0, len(colParams.AttributeGroup)*2) attributeGroupKey := fmt.Sprintf("%s_%s", attributeSet.CompTag, colParams.AttributeType) - sug := make([]redisearch.Suggestion, 0, len(colParams.AttributeGroup)*4) + sug := make([]redisearch.Suggestion, 0, len(colParams.AttributeGroup)*2+2) + configTerm := fmt.Sprintf("%s.%s", fullPath, colParams.AttributeType) + sug = append(sug, redisearch.Suggestion{ + Term: configTerm, + Score: constants.DefaultScore, + }) + if isLocalFullPath != "" { + configTerm = fmt.Sprintf("%s.%s", isLocalFullPath, colParams.AttributeType) + sug = append(sug, redisearch.Suggestion{ + Term: configTerm, + Score: constants.DefaultScore, + }) + } + for attrName, attrValue := range colParams.AttributeGroup { attrbutesGroups = append(attrbutesGroups, attrName, attrValue) attrNameMembers = append(attrNameMembers, attrName) // add redis fuzzy search suggestion for token1-token7 type - configTerm := fmt.Sprintf("%s.%s", fullPath, colParams.AttributeType) - sug = append(sug, redisearch.Suggestion{ - Term: configTerm, - Score: constants.DefaultScore, - }) - measTerm := fmt.Sprintf("%s.%s.%s", fullPath, colParams.AttributeType, attrName) sug = append(sug, redisearch.Suggestion{ Term: measTerm, @@ -140,11 +170,9 @@ func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, ful }) // add redis fuzzy search suggestion for token4-token7 type - configTerm = fmt.Sprintf("%s.%s", isLocalFullPath, colParams.AttributeType) - sug = append(sug, redisearch.Suggestion{ - Term: configTerm, - Score: constants.DefaultScore, - }) + if isLocalFullPath == "" { + continue + } measTerm = fmt.Sprintf("%s.%s.%s", isLocalFullPath, colParams.AttributeType, attrName) sug = append(sug, redisearch.Suggestion{ @@ -163,11 +191,24 @@ func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, ful } if len(sug) > 0 { - ac.AddTerms(sug...) + if err := ac.AddTerms(sug...); err != nil { + logger.Error(ctx, "add attribute group recommend suggestions failed", + "component_tag", attributeSet.CompTag, + "attribute_type", colParams.AttributeType, + "error", err, + ) + return fmt.Errorf("add attribute group recommend suggestions: %w", err) + } } _, err := pipe.Exec(ctx) if err != nil { - logger.Error(ctx, "init component attribute group recommend content failed", "error", err) + logger.Error(ctx, "init component attribute group recommend content failed", + "component_tag", attributeSet.CompTag, + "attribute_type", colParams.AttributeType, + "error", err, + ) + return fmt.Errorf("init component attribute group recommend content: %w", err) } + return nil } diff --git a/model/redis_recommend.go b/model/redis_recommend.go index 5299c96..fa0db57 100644 --- a/model/redis_recommend.go +++ b/model/redis_recommend.go @@ -36,6 +36,7 @@ func InitAutocompleterWithPool(pool *redigo.Pool) { ac = redisearch.NewAutocompleterFromPool(pool, constants.RedisSearchDictName) } +// levelOneRedisSearch 处理第一段输入的搜索, 例如 grid 或本地 nspath。 func levelOneRedisSearch(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, recommendLenType string, searchInput string, searchRedisKey string, fanInChan chan SearchResult) { defer func() { if r := recover(); r != nil { @@ -282,6 +283,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe } } +// suppressFallbackResultsWhenConcreteHitExists 有明确命中时丢弃兜底结果, 避免返回过宽推荐。 func suppressFallbackResultsWhenConcreteHitExists(results map[string]SearchResult) map[string]SearchResult { hasConcreteHit := false for _, result := range results { @@ -303,6 +305,7 @@ func suppressFallbackResultsWhenConcreteHitExists(results map[string]SearchResul return results } +// normalizeRecommendResults 统一处理 offset、去重和裁剪, 让响应只返回用户还没输入的部分。 func normalizeRecommendResults(input string, results map[string]SearchResult) map[string]SearchResult { for key, result := range results { if result.Err != nil { @@ -322,6 +325,7 @@ func normalizeRecommendResults(input string, results map[string]SearchResult) ma return results } +// calcSearchResultOffset 计算 input 与推荐项的最短公共前缀长度, 用作裁剪位置。 func calcSearchResultOffset(input string, recommends []string) int { var minOffset int for index, recommend := range recommends { @@ -333,6 +337,7 @@ func calcSearchResultOffset(input string, recommends []string) int { return minOffset } +// maxRecommendLength 返回推荐项最大长度, fallback 结果用它表示无需裁剪已有输入。 func maxRecommendLength(recommends []string) int { var maxLen int for _, recommend := range recommends { @@ -343,6 +348,7 @@ func maxRecommendLength(recommends []string) int { return maxLen } +// trimRecommendTerms 按 offset 截掉已输入前缀, 并去掉空值和重复值。 func trimRecommendTerms(recommends []string, offset int) []string { resultRecommends := make([]string, 0, len(recommends)) seen := make(map[string]struct{}) @@ -367,6 +373,7 @@ func trimRecommendTerms(recommends []string, offset int) []string { return resultRecommends } +// filterLocalNSPathResults 只保留本地组件 nspath, 避免远端 nspath 进入本地简写推荐。 func filterLocalNSPathResults(queryDatas []string) []string { filterResults := make([]string, 0, len(queryDatas)) for _, queryData := range queryDatas { @@ -390,11 +397,13 @@ func filterLocalNSPathResults(queryDatas []string) []string { return filterResults } +// queryMemberFromSpecificsLevel 根据上一层级 token 查询其真实下级集合。 func queryMemberFromSpecificsLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, keyPrefix string) ([]string, error) { queryKey := getSpecificKeyByLength(hierarchy, keyPrefix) return rdb.SMembers(ctx, queryKey).Result() } +// fallbackAllCurrentLevel 兜底返回当前层级的全量集合。 func fallbackAllCurrentLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType) ([]string, error) { setKey, ok := getAllSetKeyByHierarchy(hierarchy) if !ok { @@ -408,6 +417,61 @@ func fallbackAllCurrentLevel(ctx context.Context, rdb *redis.Client, hierarchy c return members, nil } +// fallbackCurrentLevelByInput 兜底时先校验上下级关系, 避免错误前缀返回全层级数据。 +func fallbackCurrentLevelByInput(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, inputSlice []string) ([]string, error) { + allSetKey, ok := getAllSetKeyByHierarchy(hierarchy) + if !ok { + return nil, fmt.Errorf("unsupported recommend hierarchy for fallback: %s", hierarchy) + } + + specificSetKey, ok := fallbackSpecificSetKey(hierarchy, inputSlice) + if !ok { + return fallbackAllCurrentLevel(ctx, rdb, hierarchy) + } + + count, err := rdb.SInterCard(ctx, 1, allSetKey, specificSetKey).Result() + if err != nil && err != redis.Nil { + return nil, err + } + if count == 0 { + return []string{}, nil + } + + members, err := rdb.SMembers(ctx, specificSetKey).Result() + if err != nil && err != redis.Nil { + return nil, err + } + return members, nil +} + +// fallbackSpecificSetKey 根据输入切片推导 fallback 应该校验的上一层级 specific set。 +func fallbackSpecificSetKey(hierarchy constants.RecommendHierarchyType, inputSlice []string) (string, bool) { + if len(inputSlice) == 0 { + return "", false + } + + searchInputIndex := len(inputSlice) - 1 + specificKeyIndex := searchInputIndex - 1 + if hierarchy == constants.MeasTagRecommendHierarchyType { + specificKeyIndex = searchInputIndex - 2 + } + if specificKeyIndex < 0 || specificKeyIndex >= len(inputSlice) || inputSlice[specificKeyIndex] == "" { + return "", false + } + + switch hierarchy { + case constants.ZoneRecommendHierarchyType, + constants.StationRecommendHierarchyType, + constants.CompNSPathRecommendHierarchyType, + constants.CompTagRecommendHierarchyType, + constants.MeasTagRecommendHierarchyType: + return getSpecificKeyByLength(hierarchy, inputSlice[specificKeyIndex]), true + default: + return "", false + } +} + +// getAllSetKeyByHierarchy 返回某个推荐层级对应的全量 Redis set key。 func getAllSetKeyByHierarchy(hierarchy constants.RecommendHierarchyType) (string, bool) { switch hierarchy { case constants.GridRecommendHierarchyType: @@ -429,6 +493,7 @@ func getAllSetKeyByHierarchy(hierarchy constants.RecommendHierarchyType) (string } } +// getAllKeyByGridLevel 返回所有 grid tag, 用于空输入时的初始推荐。 func getAllKeyByGridLevel(ctx context.Context, rdb *redis.Client, fanInChan chan SearchResult) { setKey := constants.RedisAllGridSetKey hierarchy := constants.GridRecommendHierarchyType @@ -453,6 +518,7 @@ func getAllKeyByGridLevel(ctx context.Context, rdb *redis.Client, fanInChan chan } } +// getAllKeyByNSPathLevel 返回所有组件 nspath, 用于空输入时的初始推荐。 func getAllKeyByNSPathLevel(ctx context.Context, rdb *redis.Client, fanInChan chan SearchResult) { queryKey := constants.RedisAllCompNSPathSetKey hierarchy := constants.CompNSPathRecommendHierarchyType @@ -477,6 +543,7 @@ func getAllKeyByNSPathLevel(ctx context.Context, rdb *redis.Client, fanInChan ch } } +// combineQueryResultByInput 把当前层级查询结果和输入前缀拼成完整推荐路径。 func combineQueryResultByInput(hierarchy constants.RecommendHierarchyType, recommendLenType string, inputSlice []string, queryResults []string) []string { prefixs := make([]string, 0, len(inputSlice)) recommandResults := make([]string, 0, len(queryResults)) @@ -521,6 +588,7 @@ func combineQueryResultByInput(hierarchy constants.RecommendHierarchyType, recom return recommandResults } +// getSpecificKeyByLength 根据层级和父级 token 生成对应的 specific set key。 func getSpecificKeyByLength(hierarchy constants.RecommendHierarchyType, keyPrefix string) string { switch hierarchy { case constants.GridRecommendHierarchyType: @@ -542,6 +610,7 @@ func getSpecificKeyByLength(hierarchy constants.RecommendHierarchyType, keyPrefi } } +// handleNSPathMeasSearch 处理 nspath.measurement 的简写搜索路径。 func handleNSPathMeasSearch(ctx context.Context, rdb *redis.Client, inputSlice []string, fanInChan chan SearchResult) { hierarchy := constants.MeasTagRecommendHierarchyType nsPath := inputSlice[0] @@ -632,6 +701,7 @@ func handleNSPathMeasSearch(ctx context.Context, rdb *redis.Client, inputSlice [ } } +// combineNSPathMeasResults 把 nspath 与 measurement tag 拼成简写推荐结果。 func combineNSPathMeasResults(nsPath string, measTags []string) []string { results := make([]string, 0, len(measTags)) for _, measTag := range measTags { @@ -640,6 +710,7 @@ func combineNSPathMeasResults(nsPath string, measTags []string) []string { return results } +// runNSPathMeasFuzzySearch 对 nspath.measurement 简写做模糊搜索, 无结果时逐字回退。 func runNSPathMeasFuzzySearch(ctx context.Context, nsPath string, searchInput string) ([]string, int, error) { searchInputLen := len([]rune(searchInput)) for searchInputLen != 0 { @@ -679,6 +750,7 @@ func runNSPathMeasFuzzySearch(ctx context.Context, nsPath string, searchInput st return []string{}, 0, nil } +// splitLastToken 拆分最后一个点号后的 token, 返回前缀和末级 token。 func splitLastToken(term string) (string, string) { lastDotIndex := strings.LastIndex(term, ".") if lastDotIndex == -1 { @@ -687,7 +759,7 @@ func splitLastToken(term string) (string, string) { return term[:lastDotIndex], term[lastDotIndex+1:] } -// handleLevelFuzzySearch define func to process recommendation logic for specific levels(level >= 2) +// handleLevelFuzzySearch 处理第二层及以后层级的精确、模糊和兜底推荐。 func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, recommendLenType string, redisSetKey string, inputSlice []string, fanInChan chan SearchResult) { inputSliceLen := len(inputSlice) searchInputIndex := inputSliceLen - 1 @@ -820,7 +892,7 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co isFallback := false if len(recommends) == 0 { logger.Info(ctx, "fuzzy search without result", "hierarchy", hierarchy, "search_input", searchInput, "error", err) - recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy) + recommends, err = fallbackCurrentLevelByInput(ctx, rdb, hierarchy, inputSlice) if err != nil { logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err) fanInChan <- SearchResult{ @@ -846,7 +918,7 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co } } -// runFuzzySearch define func to process redis fuzzy search +// runFuzzySearch 执行 RediSearch 模糊搜索, 并用 Redis set 校验候选项确属当前层级。 func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, searchPrefix string, hierarchy constants.RecommendHierarchyType, recommendLenType string) ([]string, int, error) { var configToken string var comparePrefix string @@ -904,14 +976,6 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, termLastPart = term[lastDotIndex+1:] } - if recommendLenType == constants.FullRecommendLength && hierarchy == constants.GridRecommendHierarchyType { - exists, err := rdb.SIsMember(ctx, constants.RedisAllGridSetKey, termLastPart).Result() - if err != nil || !exists { - logger.Info(ctx, "query key by redis fuzzy search failed or term not in redis all grid set", "query_key", fuzzyInput, "exists", exists, "error", err) - continue - } - } - switch recommendLenType { case constants.FullRecommendLength: if result.Term == "" { @@ -928,6 +992,20 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, } if termHierarchyLen == compareHierarchyLen && termPrefix == comparePrefix && strings.HasPrefix(termLastPart, searchInput) { + exists, err := fuzzyRecommendMemberExists(ctx, rdb, hierarchy, recommendLenType, comparePrefix, termLastPart) + if err != nil || !exists { + logger.Info(ctx, "fuzzy recommend term not found in specific redis set", + "hierarchy", hierarchy, + "recommend_len_type", recommendLenType, + "query_key", fuzzyInput, + "term", term, + "term_last_part", termLastPart, + "exists", exists, + "error", err, + ) + continue + } + recommend := result.Term if hierarchy == constants.MeasTagRecommendHierarchyType { recommend = strings.Join([]string{termPrefix, configToken, termLastPart}, ".") @@ -942,11 +1020,59 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, continue } - return recommends, len([]rune(searchInput)), nil + return recommends, fuzzyRecommendOffset(searchPrefix, searchInput), nil } return []string{}, 0, nil } +// fuzzyRecommendOffset 返回模糊命中的裁剪位置, 保留完整输入前缀参与 offset。 +func fuzzyRecommendOffset(searchPrefix string, searchInput string) int { + if searchPrefix == "" { + return len([]rune(searchInput)) + } + return len([]rune(searchPrefix)) + 1 + len([]rune(searchInput)) +} + +// fuzzyRecommendMemberExists 校验模糊候选末级 token 是否存在于对应 Redis set。 +func fuzzyRecommendMemberExists(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, recommendLenType string, comparePrefix string, termLastPart string) (bool, error) { + setKey, ok := fuzzyRecommendMemberSetKey(hierarchy, recommendLenType, comparePrefix) + if !ok { + return true, nil + } + return rdb.SIsMember(ctx, setKey, termLastPart).Result() +} + +// fuzzyRecommendMemberSetKey 返回模糊候选校验时应使用的 Redis set key。 +func fuzzyRecommendMemberSetKey(hierarchy constants.RecommendHierarchyType, recommendLenType string, comparePrefix string) (string, bool) { + switch hierarchy { + case constants.GridRecommendHierarchyType: + return constants.RedisAllGridSetKey, true + case constants.ZoneRecommendHierarchyType: + return fmt.Sprintf(constants.RedisSpecGridZoneSetKey, comparePrefix), comparePrefix != "" + case constants.StationRecommendHierarchyType: + return fmt.Sprintf(constants.RedisSpecZoneStationSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != "" + case constants.CompNSPathRecommendHierarchyType: + return fmt.Sprintf(constants.RedisSpecStationCompNSPATHSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != "" + case constants.CompTagRecommendHierarchyType: + return fmt.Sprintf(constants.RedisSpecCompNSPathCompTagSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != "" + case constants.ConfigRecommendHierarchyType: + return constants.RedisAllConfigSetKey, true + case constants.MeasTagRecommendHierarchyType: + return fmt.Sprintf(constants.RedisSpecCompTagMeasSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != "" + default: + return "", false + } +} + +// lastRecommendPrefixToken 取推荐前缀中的最后一个层级 token。 +func lastRecommendPrefixToken(prefix string) string { + if lastDotIndex := strings.LastIndex(prefix, "."); lastDotIndex != -1 { + return prefix[lastDotIndex+1:] + } + return prefix +} + +// trimLastRune 删除字符串最后一个 rune, 用于模糊搜索逐字回退。 func trimLastRune(s string) string { runes := []rune(s) if len(runes) == 0 { diff --git a/model/redis_recommend_test.go b/model/redis_recommend_test.go new file mode 100644 index 0000000..259aee0 --- /dev/null +++ b/model/redis_recommend_test.go @@ -0,0 +1,40 @@ +package model + +import "testing" + +func TestFuzzyRecommendOffsetUsesFullMatchedInput(t *testing.T) { + tests := []struct { + name string + searchPrefix string + searchInput string + want int + }{ + { + name: "zone fuzzy prefix", + searchPrefix: "grid000", + searchInput: "z", + want: len([]rune("grid000.z")), + }, + { + name: "level one fuzzy", + searchPrefix: "", + searchInput: "g", + want: len([]rune("g")), + }, + { + name: "measurement fuzzy preserves config token", + searchPrefix: "grid.zone.station.nspath.comp.config", + searchInput: "m", + want: len([]rune("grid.zone.station.nspath.comp.config.m")), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := fuzzyRecommendOffset(tt.searchPrefix, tt.searchInput) + if got != tt.want { + t.Fatalf("expected offset %d, got %d", tt.want, got) + } + }) + } +}