fix: improve measurement recommend fallback and rune offsets
- fallback to all current-level candidates when fuzzy search has no result - track fallback results and calculate fallback offsets from max rune length - use rune-based prefix offsets and trimming for non-ASCII tokens - reuse local nspath filtering for component nspath recommendations
This commit is contained in:
parent
367de31247
commit
66870c7008
|
|
@ -24,6 +24,7 @@ type SearchResult struct {
|
|||
RecommendType constants.RecommendHierarchyType
|
||||
QueryDatas []string
|
||||
IsFuzzy bool
|
||||
IsFallback bool
|
||||
Offset int
|
||||
Err error
|
||||
}
|
||||
|
|
@ -96,11 +97,25 @@ func levelOneRedisSearch(ctx context.Context, rdb *redis.Client, hierarchy const
|
|||
return
|
||||
}
|
||||
|
||||
recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
|
||||
fanInChan <- SearchResult{
|
||||
RecommendType: hierarchy,
|
||||
QueryDatas: nil,
|
||||
IsFuzzy: true,
|
||||
IsFallback: true,
|
||||
Err: err,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fanInChan <- SearchResult{
|
||||
RecommendType: hierarchy,
|
||||
QueryDatas: []string{},
|
||||
QueryDatas: recommends,
|
||||
IsFuzzy: true,
|
||||
Offset: offset,
|
||||
IsFallback: true,
|
||||
Offset: 0,
|
||||
Err: nil,
|
||||
}
|
||||
}
|
||||
|
|
@ -127,21 +142,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe
|
|||
}
|
||||
|
||||
if result.RecommendType == constants.CompNSPathRecommendHierarchyType {
|
||||
filterResults := make([]string, 0, len(result.QueryDatas))
|
||||
// TODO 增加 nspath 过滤
|
||||
for _, queryData := range result.QueryDatas {
|
||||
var nsPath string
|
||||
if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
|
||||
nsPath = queryData
|
||||
} else {
|
||||
nsPath = queryData[lastDotIndex+1:]
|
||||
}
|
||||
|
||||
if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
|
||||
filterResults = append(filterResults, queryData)
|
||||
}
|
||||
}
|
||||
result.QueryDatas = filterResults
|
||||
result.QueryDatas = filterLocalNSPathResults(result.QueryDatas)
|
||||
}
|
||||
results[result.RecommendType.String()] = result
|
||||
}
|
||||
|
|
@ -171,26 +172,7 @@ func RedisSearchRecommend(ctx context.Context, input string) map[string]SearchRe
|
|||
}
|
||||
|
||||
if result.RecommendType == constants.CompNSPathRecommendHierarchyType {
|
||||
filterResults := make([]string, 0, len(result.QueryDatas))
|
||||
// TODO 增加 nspath 过滤
|
||||
for _, queryData := range result.QueryDatas {
|
||||
if queryData == "." {
|
||||
filterResults = append(filterResults, queryData)
|
||||
continue
|
||||
}
|
||||
|
||||
var nsPath string
|
||||
if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
|
||||
nsPath = queryData
|
||||
} else {
|
||||
nsPath = queryData[lastDotIndex+1:]
|
||||
}
|
||||
|
||||
if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
|
||||
filterResults = append(filterResults, queryData)
|
||||
}
|
||||
}
|
||||
result.QueryDatas = filterResults
|
||||
result.QueryDatas = filterLocalNSPathResults(result.QueryDatas)
|
||||
}
|
||||
results[result.RecommendType.String()] = result
|
||||
}
|
||||
|
|
@ -302,6 +284,11 @@ func normalizeRecommendResults(input string, results map[string]SearchResult) ma
|
|||
if result.Err != nil {
|
||||
continue
|
||||
}
|
||||
if result.IsFallback {
|
||||
result.Offset = maxRecommendLength(result.QueryDatas)
|
||||
results[key] = result
|
||||
continue
|
||||
}
|
||||
if !result.IsFuzzy {
|
||||
result.Offset = calcSearchResultOffset(input, result.QueryDatas)
|
||||
}
|
||||
|
|
@ -322,16 +309,27 @@ func calcSearchResultOffset(input string, recommends []string) int {
|
|||
return minOffset
|
||||
}
|
||||
|
||||
func maxRecommendLength(recommends []string) int {
|
||||
var maxLen int
|
||||
for _, recommend := range recommends {
|
||||
if length := len([]rune(recommend)); length > maxLen {
|
||||
maxLen = length
|
||||
}
|
||||
}
|
||||
return maxLen
|
||||
}
|
||||
|
||||
func trimRecommendTerms(recommends []string, offset int) []string {
|
||||
resultRecommends := make([]string, 0, len(recommends))
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
for _, recommend := range recommends {
|
||||
if offset > len(recommend) {
|
||||
recommendRunes := []rune(recommend)
|
||||
if offset > len(recommendRunes) {
|
||||
continue
|
||||
}
|
||||
|
||||
recommendTerm := recommend[offset:]
|
||||
recommendTerm := string(recommendRunes[offset:])
|
||||
if recommendTerm == "" {
|
||||
continue
|
||||
}
|
||||
|
|
@ -345,11 +343,68 @@ func trimRecommendTerms(recommends []string, offset int) []string {
|
|||
return resultRecommends
|
||||
}
|
||||
|
||||
func filterLocalNSPathResults(queryDatas []string) []string {
|
||||
filterResults := make([]string, 0, len(queryDatas))
|
||||
for _, queryData := range queryDatas {
|
||||
if queryData == "." {
|
||||
filterResults = append(filterResults, queryData)
|
||||
continue
|
||||
}
|
||||
|
||||
var nsPath string
|
||||
if lastDotIndex := strings.LastIndex(queryData, "."); lastDotIndex == -1 {
|
||||
nsPath = queryData
|
||||
} else {
|
||||
nsPath = queryData[lastDotIndex+1:]
|
||||
}
|
||||
|
||||
// TODO 考虑为 NSPathToIsLocalMap 增加读写锁或改为并发安全快照, 保证过滤数据一致性
|
||||
if isLocal, ok := NSPathToIsLocalMap[nsPath]; ok && isLocal {
|
||||
filterResults = append(filterResults, queryData)
|
||||
}
|
||||
}
|
||||
return filterResults
|
||||
}
|
||||
|
||||
func queryMemberFromSpecificsLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType, keyPrefix string) ([]string, error) {
|
||||
queryKey := getSpecificKeyByLength(hierarchy, keyPrefix)
|
||||
return rdb.SMembers(ctx, queryKey).Result()
|
||||
}
|
||||
|
||||
func fallbackAllCurrentLevel(ctx context.Context, rdb *redis.Client, hierarchy constants.RecommendHierarchyType) ([]string, error) {
|
||||
setKey, ok := getAllSetKeyByHierarchy(hierarchy)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported recommend hierarchy for fallback: %s", hierarchy)
|
||||
}
|
||||
|
||||
members, err := rdb.SMembers(ctx, setKey).Result()
|
||||
if err != nil && err != redis.Nil {
|
||||
return nil, err
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func getAllSetKeyByHierarchy(hierarchy constants.RecommendHierarchyType) (string, bool) {
|
||||
switch hierarchy {
|
||||
case constants.GridRecommendHierarchyType:
|
||||
return constants.RedisAllGridSetKey, true
|
||||
case constants.ZoneRecommendHierarchyType:
|
||||
return constants.RedisAllZoneSetKey, true
|
||||
case constants.StationRecommendHierarchyType:
|
||||
return constants.RedisAllStationSetKey, true
|
||||
case constants.CompNSPathRecommendHierarchyType:
|
||||
return constants.RedisAllCompNSPathSetKey, true
|
||||
case constants.CompTagRecommendHierarchyType:
|
||||
return constants.RedisAllCompTagSetKey, true
|
||||
case constants.ConfigRecommendHierarchyType:
|
||||
return constants.RedisAllConfigSetKey, true
|
||||
case constants.MeasTagRecommendHierarchyType:
|
||||
return constants.RedisAllMeasTagSetKey, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func getAllKeyByGridLevel(ctx context.Context, rdb *redis.Client, fanInChan chan SearchResult) {
|
||||
setKey := constants.RedisAllGridSetKey
|
||||
hierarchy := constants.GridRecommendHierarchyType
|
||||
|
|
@ -593,14 +648,30 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
|
|||
return
|
||||
}
|
||||
|
||||
isFallback := false
|
||||
if len(recommends) == 0 {
|
||||
logger.Info(ctx, "fuzzy search without result", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
|
||||
recommends, err = fallbackAllCurrentLevel(ctx, rdb, hierarchy)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "fallback all keys at current level failed", "hierarchy", hierarchy, "search_input", searchInput, "error", err)
|
||||
fanInChan <- SearchResult{
|
||||
RecommendType: hierarchy,
|
||||
QueryDatas: nil,
|
||||
IsFuzzy: true,
|
||||
IsFallback: true,
|
||||
Err: err,
|
||||
}
|
||||
return
|
||||
}
|
||||
isFallback = true
|
||||
offset = 0
|
||||
}
|
||||
|
||||
fanInChan <- SearchResult{
|
||||
RecommendType: hierarchy,
|
||||
QueryDatas: recommends,
|
||||
IsFuzzy: true,
|
||||
IsFallback: isFallback,
|
||||
Offset: offset,
|
||||
Err: nil,
|
||||
}
|
||||
|
|
@ -610,7 +681,7 @@ func handleLevelFuzzySearch(ctx context.Context, rdb *redis.Client, hierarchy co
|
|||
func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string, searchPrefix string, hierarchy constants.RecommendHierarchyType, recommendLenType string) ([]string, int, error) {
|
||||
var configToken string
|
||||
var comparePrefix string
|
||||
searchInputLen := len(searchInput)
|
||||
searchInputLen := len([]rune(searchInput))
|
||||
compareHierarchyLen := int(hierarchy)
|
||||
comparePrefix = searchPrefix
|
||||
if hierarchy == constants.MeasTagRecommendHierarchyType {
|
||||
|
|
@ -641,10 +712,10 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
|
|||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
// 如果没有结果，退一步（删除最后一个字节）并继续循环
|
||||
// TODO 考虑使用其他方式代替 for 循环退一字节的查询方式
|
||||
searchInput = searchInput[:len(searchInput)-1]
|
||||
searchInputLen = len(searchInput)
|
||||
// 如果没有结果，退一步（删除最后一个字符）并继续循环
|
||||
// TODO 考虑使用其他方式代替 for 循环退一字符的查询方式
|
||||
searchInput = trimLastRune(searchInput)
|
||||
searchInputLen = len([]rune(searchInput))
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -697,12 +768,20 @@ func runFuzzySearch(ctx context.Context, rdb *redis.Client, searchInput string,
|
|||
}
|
||||
|
||||
if len(recommends) == 0 {
|
||||
searchInput = searchInput[:len(searchInput)-1]
|
||||
searchInputLen = len(searchInput)
|
||||
searchInput = trimLastRune(searchInput)
|
||||
searchInputLen = len([]rune(searchInput))
|
||||
continue
|
||||
}
|
||||
|
||||
return recommends, len(searchInput), nil
|
||||
return recommends, len([]rune(searchInput)), nil
|
||||
}
|
||||
return []string{}, 0, nil
|
||||
}
|
||||
|
||||
func trimLastRune(s string) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) == 0 {
|
||||
return s
|
||||
}
|
||||
return string(runes[:len(runes)-1])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,10 +46,12 @@ func GetLongestCommonPrefixLength(query string, result string) int {
|
|||
return 0
|
||||
}
|
||||
|
||||
minLen := min(len(query), len(result))
|
||||
queryRunes := []rune(query)
|
||||
resultRunes := []rune(result)
|
||||
minLen := min(len(queryRunes), len(resultRunes))
|
||||
|
||||
for i := range minLen {
|
||||
if query[i] != result[i] {
|
||||
if queryRunes[i] != resultRunes[i] {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue