CL-Softwares
/
modelRT
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: improve redis recommendation fuzzy matching 

- add bounded errgroup handling for attribute group suggestion initialization
- add config-level suggestions and propagate Redis/RediSearch write errors
- constrain fuzzy recommendation candidates by hierarchy-specific Redis sets
- avoid broad fallback results when concrete matches exist
- preserve full matched prefix when calculating fuzzy recommendation offset
- add tests for fuzzy recommendation offset calculation
This commit is contained in:
douxu 2026-07-07 17:11:25 +08:00
parent 305bdd4dcf
commit b53746efcd
3 changed files with 236 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
model/attribute_group_recommend_model.go
View File

@ -12,6 +12,7 @@ import (
"modelRT/orm"
"github.com/RediSearch/redisearch-go/v2/redisearch"
"golang.org/x/sync/errgroup"
"gorm.io/gorm"
)
@ -20,6 +21,13 @@ type columnParam struct {
AttributeGroup map[string]any
}
type attributeGroupRecommendJob struct {
AttributeSet orm.AttributeSet
FullPath string
IsLocalFullPath string
ColumnParam columnParam
}
// TraverseAttributeGroupTables define func to traverse component attribute group tables
func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compTagToFullPath map[string]string, isLocalCompTagToFullPath map[string]string, compAttrSet map[string]orm.AttributeSet) error {
var tableNames []string
@ -38,6 +46,7 @@ func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compTagToFul
return nil
}
jobs := make([]attributeGroupRecommendJob, 0)
for _, tableName := range tableNames {
var records []map[string]any
err := db.Table(tableName).Find(&records).Error
@ -102,13 +111,27 @@ func TraverseAttributeGroupTables(ctx context.Context, db *gorm.DB, compTagToFul
AttributeType: attributeType,
AttributeGroup: attributeGroup,
}
go storeAttributeGroup(ctx, attrSet, fullPath, isLocalfullPath, columnParam)
jobs = append(jobs, attributeGroupRecommendJob{
AttributeSet: attrSet,
FullPath: fullPath,
IsLocalFullPath: isLocalfullPath,
ColumnParam: columnParam,
})
}
}
return nil
group, groupCtx := errgroup.WithContext(ctx)
group.SetLimit(16)
for _, job := range jobs {
job := job
group.Go(func() error {
return storeAttributeGroup(groupCtx, job.AttributeSet, job.FullPath, job.IsLocalFullPath, job.ColumnParam)
})
}
return group.Wait()
}
func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, fullPath string, isLocalFullPath string, colParams columnParam) {
func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, fullPath string, isLocalFullPath string, colParams columnParam) error {
rdb := diagram.GetRedisClientInstance()
pipe := rdb.Pipeline()
@ -121,18 +144,25 @@ func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, ful
attrbutesGroups := make([]any, 0, len(colParams.AttributeGroup)*2)
attributeGroupKey := fmt.Sprintf("%s_%s", attributeSet.CompTag, colParams.AttributeType)
sug := make([]redisearch.Suggestion, 0, len(colParams.AttributeGroup)*4)
sug := make([]redisearch.Suggestion, 0, len(colParams.AttributeGroup)*2+2)
configTerm := fmt.Sprintf("%s.%s", fullPath, colParams.AttributeType)
sug = append(sug, redisearch.Suggestion{
Term: configTerm,
Score: constants.DefaultScore,
})
if isLocalFullPath != "" {
configTerm = fmt.Sprintf("%s.%s", isLocalFullPath, colParams.AttributeType)
sug = append(sug, redisearch.Suggestion{
Term: configTerm,
Score: constants.DefaultScore,
})
}
for attrName, attrValue := range colParams.AttributeGroup {
attrbutesGroups = append(attrbutesGroups, attrName, attrValue)
attrNameMembers = append(attrNameMembers, attrName)
// add redis fuzzy search suggestion for token1-token7 type
configTerm := fmt.Sprintf("%s.%s", fullPath, colParams.AttributeType)
sug = append(sug, redisearch.Suggestion{
Term: configTerm,
Score: constants.DefaultScore,
})
measTerm := fmt.Sprintf("%s.%s.%s", fullPath, colParams.AttributeType, attrName)
sug = append(sug, redisearch.Suggestion{
Term: measTerm,
@ -140,11 +170,9 @@ func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, ful
})
// add redis fuzzy search suggestion for token4-token7 type
configTerm = fmt.Sprintf("%s.%s", isLocalFullPath, colParams.AttributeType)
sug = append(sug, redisearch.Suggestion{
Term: configTerm,
Score: constants.DefaultScore,
})
if isLocalFullPath == "" {
continue
}
measTerm = fmt.Sprintf("%s.%s.%s", isLocalFullPath, colParams.AttributeType, attrName)
sug = append(sug, redisearch.Suggestion{
@ -163,11 +191,24 @@ func storeAttributeGroup(ctx context.Context, attributeSet orm.AttributeSet, ful
}
if len(sug) > 0 {
ac.AddTerms(sug...)
if err := ac.AddTerms(sug...); err != nil {
logger.Error(ctx, "add attribute group recommend suggestions failed",
"component_tag", attributeSet.CompTag,
"attribute_type", colParams.AttributeType,
"error", err,
)
return fmt.Errorf("add attribute group recommend suggestions: %w", err)
}
}
_, err := pipe.Exec(ctx)
if err != nil {
logger.Error(ctx, "init component attribute group recommend content failed", "error", err)
logger.Error(ctx, "init component attribute group recommend content failed",
"component_tag", attributeSet.CompTag,
"attribute_type", colParams.AttributeType,
"error", err,
)
return fmt.Errorf("init component attribute group recommend content: %w", err)
}
return nil
}

150
model/redis_recommend.go
View File

@ -36,6 +36,7 @@ func InitAutocompleterWithPool(pool *redigo.Pool) {
ac = redisearch.NewAutocompleterFromPool(pool, constants.RedisSearchDictName)
}
// levelOneRedisSearch 处理第一段输入的搜索, 例如 grid 或本地 nspath。
func levelOneRedisSearch(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, recommendLenType string, searchInput string, searchRedisKey string, fanInChan chan SearchResult) {
defer func() {
if r := recover(); r != nil {
@ -282,6 +283,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe
}
}
// suppressFallbackResultsWhenConcreteHitExists 有明确命中时丢弃兜底结果, 避免返回过宽推荐。
func suppressFallbackResultsWhenConcreteHitExists(results map[string]SearchResult) map[string]SearchResult {
hasConcreteHit := false
for _, result := range results {
@ -303,6 +305,7 @@ func suppressFallbackResultsWhenConcreteHitExists(results map[string]SearchResul
return results
}
// normalizeRecommendResults 统一处理 offset、去重和裁剪, 让响应只返回用户还没输入的部分。
func normalizeRecommendResults(input string, results map[string]SearchResult) map[string]SearchResult {
for key, result := range results {
if result.Err != nil {
@ -322,6 +325,7 @@ func normalizeRecommendResults(input string, results map[string]SearchResult) ma
return results
}
// calcSearchResultOffset 计算 input 与推荐项的最短公共前缀长度, 用作裁剪位置。
func calcSearchResultOffset(input string, recommends []string) int {
var minOffset int
for index, recommend := range recommends {
@ -333,6 +337,7 @@ func calcSearchResultOffset(input string, recommends []string) int {
return minOffset
}
// maxRecommendLength 返回推荐项最大长度, fallback 结果用它表示无需裁剪已有输入。
func maxRecommendLength(recommends []string) int {
var maxLen int
for _, recommend := range recommends {
@ -343,6 +348,7 @@ func maxRecommendLength(recommends []string) int {
return maxLen
}
// trimRecommendTerms 按 offset 截掉已输入前缀, 并去掉空值和重复值。
func trimRecommendTerms(recommends []string, offset int) []string {
resultRecommends := make([]string, 0, len(recommends))
seen := make(map[string]struct{})
@ -367,6 +373,7 @@ func trimRecommendTerms(recommends []string, offset int) []string {
return resultRecommends
}
// filterLocalNSPathResults 只保留本地组件 nspath, 避免远端 nspath 进入本地简写推荐。
func filterLocalNSPathResults(queryDatas []string) []string {
filterResults := make([]string, 0, len(queryDatas))
for _, queryData := range queryDatas {
@ -390,11 +397,13 @@ func filterLocalNSPathResults(queryDatas []string) []string {
return filterResults
}
// queryMemberFromSpecificsLevel 根据上一层级 token 查询其真实下级集合。
func queryMemberFromSpecificsLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, keyPrefix string) ([]string, error) {
queryKey := getSpecificKeyByLength(hierarchy, keyPrefix)
return rdb.SMembers(ctx, queryKey).Result()
}
// fallbackAllCurrentLevel 兜底返回当前层级的全量集合。
func fallbackAllCurrentLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType) ([]string, error) {
setKey, ok := getAllSetKeyByHierarchy(hierarchy)
if !ok {
@ -408,6 +417,61 @@ func fallbackAllCurrentLevel(ctx context.Context, rdb *redis.Client, hierarchy c
return members, nil
}
// fallbackCurrentLevelByInput 兜底时先校验上下级关系, 避免错误前缀返回全层级数据。
func fallbackCurrentLevelByInput(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, inputSlice []string) ([]string, error) {
allSetKey, ok := getAllSetKeyByHierarchy(hierarchy)
if !ok {
return nil, fmt.Errorf("unsupported recommend hierarchy for fallback: %s", hierarchy)
}
specificSetKey, ok := fallbackSpecificSetKey(hierarchy, inputSlice)
if !ok {
return fallbackAllCurrentLevel(ctx, rdb, hierarchy)
}
count, err := rdb.SInterCard(ctx, 1, allSetKey, specificSetKey).Result()
if err != nil && err != redis.Nil {
return nil, err
}
if count == 0 {
return []string{}, nil
}
members, err := rdb.SMembers(ctx, specificSetKey).Result()
if err != nil && err != redis.Nil {
return nil, err
}
return members, nil
}
// fallbackSpecificSetKey 根据输入切片推导 fallback 应该校验的上一层级 specific set。
func fallbackSpecificSetKey(hierarchy constants.RecommendHierarchyType, inputSlice []string) (string, bool) {
if len(inputSlice) == 0 {
return "", false
}
searchInputIndex := len(inputSlice) - 1
specificKeyIndex := searchInputIndex - 1
if hierarchy == constants.MeasTagRecommendHierarchyType {
specificKeyIndex = searchInputIndex - 2
}
if specificKeyIndex < 0 || specificKeyIndex >= len(inputSlice) || inputSlice[specificKeyIndex] == "" {
return "", false
}
switch hierarchy {
case constants.ZoneRecommendHierarchyType,
constants.StationRecommendHierarchyType,
constants.CompNSPathRecommendHierarchyType,
constants.CompTagRecommendHierarchyType,
constants.MeasTagRecommendHierarchyType:
return getSpecificKeyByLength(hierarchy, inputSlice[specificKeyIndex]), true
default:
return "", false
}
}
// getAllSetKeyByHierarchy 返回某个推荐层级对应的全量 Redis set key。
func getAllSetKeyByHierarchy(hierarchy constants.RecommendHierarchyType) (string, bool) {
switch hierarchy {
case constants.GridRecommendHierarchyType:
@ -429,6 +493,7 @@ func getAllSetKeyByHierarchy(hierarchy constants.RecommendHierarchyType) (string
}
}
// getAllKeyByGridLevel 返回所有 grid tag, 用于空输入时的初始推荐。
func getAllKeyByGridLevel(ctx context.Context, rdb *redis.Client, fanInChan chan SearchResult) {
setKey := constants.RedisAllGridSetKey
hierarchy := constants.GridRecommendHierarchyType
@ -453,6 +518,7 @@ func getAllKeyByGridLevel(ctx context.Context, rdb *redis.Client, fanInChan chan
}
}
// getAllKeyByNSPathLevel 返回所有组件 nspath, 用于空输入时的初始推荐。
func getAllKeyByNSPathLevel(ctx context.Context, rdb *redis.Client, fanInChan chan SearchResult) {
queryKey := constants.RedisAllCompNSPathSetKey
hierarchy := constants.CompNSPathRecommendHierarchyType
@ -477,6 +543,7 @@ func getAllKeyByNSPathLevel(ctx context.Context, rdb *redis.Client, fanInChan ch
}
}
// combineQueryResultByInput 把当前层级查询结果和输入前缀拼成完整推荐路径。
func combineQueryResultByInput(hierarchy constants.RecommendHierarchyType, recommendLenType string, inputSlice []string, queryResults []string) []string {
prefixs := make([]string, 0, len(inputSlice))
recommandResults := make([]string, 0, len(queryResults))
@ -521,6 +588,7 @@ func combineQueryResultByInput(hierarchy constants.RecommendHierarchyType, recom
return recommandResults
}
// getSpecificKeyByLength 根据层级和父级 token 生成对应的 specific set key。
func getSpecificKeyByLength(hierarchy constants.RecommendHierarchyType, keyPrefix string) string {
switch hierarchy {
case constants.GridRecommendHierarchyType:
@ -542,6 +610,7 @@ func getSpecificKeyByLength(hierarchy constants.RecommendHierarchyType, keyPrefi
}
}
// handleNSPathMeasSearch 处理 nspath.measurement 的简写搜索路径。
func handleNSPathMeasSearch(ctx context.Context, rdb *redis.Client, inputSlice []string, fanInChan chan SearchResult) {
hierarchy := constants.MeasTagRecommendHierarchyType
nsPath := inputSlice[0]
@ -632,6 +701,7 @@ func handleNSPathMeasSearch(ctx context.Context, rdb *redis.Client, inputSlice [
}
}
// combineNSPathMeasResults 把 nspath 与 measurement tag 拼成简写推荐结果。
func combineNSPathMeasResults(nsPath string, measTags []string) []string {
results := make([]string, 0, len(measTags))
for _, measTag := range measTags {
@ -640,6 +710,7 @@ func combineNSPathMeasResults(nsPath string, measTags []string) []string {
return results
}
// runNSPathMeasFuzzySearch 对 nspath.measurement 简写做模糊搜索, 无结果时逐字回退。
func runNSPathMeasFuzzySearch(ctx context.Context, nsPath string, searchInput string) ([]string, int, error) {
searchInputLen := len([]rune(searchInput))
for searchInputLen != 0 {
@ -679,6 +750,7 @@ func runNSPathMeasFuzzySearch(ctx context.Context, nsPath string, searchInput st
return []string{}, 0, nil
}
// splitLastToken 拆分最后一个点号后的 token, 返回前缀和末级 token。
func splitLastToken(term string) (string, string) {
lastDotIndex := strings.LastIndex(term, ".")
if lastDotIndex == -1 {
@ -687,7 +759,7 @@ func splitLastToken(term string) (string, string) {
return term[:lastDotIndex], term[lastDotIndex+1:]
}
// handleLevelFuzzySearch define func to process recommendation logic for specific levels(level >= 2)
// handleLevelFuzzySearch 处理第二层及以后层级的精确、模糊和兜底推荐。
func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, recommendLenType string, redisSetKey string, inputSlice []string, fanInChan chan SearchResult) {
inputSliceLen := len(inputSlice)
searchInputIndex := inputSliceLen - 1
@ -820,7 +892,7 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
isFallback := false
if len(recommends) == 0 {
logger.Info(ctx, "fuzzy search without result", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy)
recommends, err = fallbackCurrentLevelByInput(ctx, rdb, hierarchy, inputSlice)
if err != nil {
logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
fanInChan <- SearchResult{
@ -846,7 +918,7 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
}
}
// runFuzzySearch define func to process redis fuzzy search
// runFuzzySearch 执行 RediSearch 模糊搜索, 并用 Redis set 校验候选项确属当前层级。
func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, searchPrefix string, hierarchy constants.RecommendHierarchyType, recommendLenType string) ([]string, int, error) {
var configToken string
var comparePrefix string
@ -904,14 +976,6 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
termLastPart = term[lastDotIndex+1:]
}
if recommendLenType == constants.FullRecommendLength && hierarchy == constants.GridRecommendHierarchyType {
exists, err := rdb.SIsMember(ctx, constants.RedisAllGridSetKey, termLastPart).Result()
if err != nil || !exists {
logger.Info(ctx, "query key by redis fuzzy search failed or term not in redis all grid set", "query_key", fuzzyInput, "exists", exists, "error", err)
continue
}
}
switch recommendLenType {
case constants.FullRecommendLength:
if result.Term == "" {
@ -928,6 +992,20 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
}
if termHierarchyLen == compareHierarchyLen && termPrefix == comparePrefix && strings.HasPrefix(termLastPart, searchInput) {
exists, err := fuzzyRecommendMemberExists(ctx, rdb, hierarchy, recommendLenType, comparePrefix, termLastPart)
if err != nil || !exists {
logger.Info(ctx, "fuzzy recommend term not found in specific redis set",
"hierarchy", hierarchy,
"recommend_len_type", recommendLenType,
"query_key", fuzzyInput,
"term", term,
"term_last_part", termLastPart,
"exists", exists,
"error", err,
)
continue
}
recommend := result.Term
if hierarchy == constants.MeasTagRecommendHierarchyType {
recommend = strings.Join([]string{termPrefix, configToken, termLastPart}, ".")
@ -942,11 +1020,59 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
continue
}
return recommends, len([]rune(searchInput)), nil
return recommends, fuzzyRecommendOffset(searchPrefix, searchInput), nil
}
return []string{}, 0, nil
}
// fuzzyRecommendOffset 返回模糊命中的裁剪位置, 保留完整输入前缀参与 offset。
func fuzzyRecommendOffset(searchPrefix string, searchInput string) int {
if searchPrefix == "" {
return len([]rune(searchInput))
}
return len([]rune(searchPrefix)) + 1 + len([]rune(searchInput))
}
// fuzzyRecommendMemberExists 校验模糊候选末级 token 是否存在于对应 Redis set。
func fuzzyRecommendMemberExists(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, recommendLenType string, comparePrefix string, termLastPart string) (bool, error) {
setKey, ok := fuzzyRecommendMemberSetKey(hierarchy, recommendLenType, comparePrefix)
if !ok {
return true, nil
}
return rdb.SIsMember(ctx, setKey, termLastPart).Result()
}
// fuzzyRecommendMemberSetKey 返回模糊候选校验时应使用的 Redis set key。
func fuzzyRecommendMemberSetKey(hierarchy constants.RecommendHierarchyType, recommendLenType string, comparePrefix string) (string, bool) {
switch hierarchy {
case constants.GridRecommendHierarchyType:
return constants.RedisAllGridSetKey, true
case constants.ZoneRecommendHierarchyType:
return fmt.Sprintf(constants.RedisSpecGridZoneSetKey, comparePrefix), comparePrefix != ""
case constants.StationRecommendHierarchyType:
return fmt.Sprintf(constants.RedisSpecZoneStationSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != ""
case constants.CompNSPathRecommendHierarchyType:
return fmt.Sprintf(constants.RedisSpecStationCompNSPATHSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != ""
case constants.CompTagRecommendHierarchyType:
return fmt.Sprintf(constants.RedisSpecCompNSPathCompTagSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != ""
case constants.ConfigRecommendHierarchyType:
return constants.RedisAllConfigSetKey, true
case constants.MeasTagRecommendHierarchyType:
return fmt.Sprintf(constants.RedisSpecCompTagMeasSetKey, lastRecommendPrefixToken(comparePrefix)), comparePrefix != ""
default:
return "", false
}
}
// lastRecommendPrefixToken 取推荐前缀中的最后一个层级 token。
func lastRecommendPrefixToken(prefix string) string {
if lastDotIndex := strings.LastIndex(prefix, "."); lastDotIndex != -1 {
return prefix[lastDotIndex+1:]
}
return prefix
}
// trimLastRune 删除字符串最后一个 rune, 用于模糊搜索逐字回退。
func trimLastRune(s string) string {
runes := []rune(s)
if len(runes) == 0 {

40
model/redis_recommend_test.go Normal file
View File

@ -0,0 +1,40 @@
package model
import "testing"
func TestFuzzyRecommendOffsetUsesFullMatchedInput(t *testing.T) {
tests := []struct {
name string
searchPrefix string
searchInput string
want int
}{
{
name: "zone fuzzy prefix",
searchPrefix: "grid000",
searchInput: "z",
want: len([]rune("grid000.z")),
},
{
name: "level one fuzzy",
searchPrefix: "",
searchInput: "g",
want: len([]rune("g")),
},
{
name: "measurement fuzzy preserves config token",
searchPrefix: "grid.zone.station.nspath.comp.config",
searchInput: "m",
want: len([]rune("grid.zone.station.nspath.comp.config.m")),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := fuzzyRecommendOffset(tt.searchPrefix, tt.searchInput)
if got != tt.want {
t.Fatalf("expected offset %d, got %d", tt.want, got)
}
})
}
}