CL-Softwares
/
modelRT
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

optimize measurement recommend api

This commit is contained in:
douxu 2025-10-16 17:18:57 +08:00
parent 62e897190d
commit f0a66263a3
4 changed files with 120 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -22,3 +22,5 @@
go.work
.vscode
# Shield all log files in the log folder
/log/

2
config/config.yaml
View File

@ -25,7 +25,7 @@ kafka:
logger:
mode: "development"
level: "debug"
filepath: "/home/douxu/log/modelRT-%s.log"
filepath: "/Users/douxu/Workspace/coslight/modelRT/modelRT-%s.log"
maxsize: 1
maxbackups: 5
maxage: 30

46
handler/measurement_recommend.go
View File

@ -2,6 +2,7 @@
package handler
import (
"fmt"
"net/http"
"modelRT/logger"
@ -24,19 +25,34 @@ func MeasurementRecommendHandler(c *gin.Context) {
return
}
recommends, err := model.RedisSearchRecommend(c, request.Input)
recommends, isFuzzy, err := model.RedisSearchRecommend(c, request.Input)
// TODO delete debug info
fmt.Printf("recommends in handler:%+v\n", recommends)
if err != nil {
logger.Error(c, "failed to get recommend data from redis", "input", request.Input, "error", err)
c.JSON(http.StatusOK, network.FailureResponse{
Code: http.StatusInternalServerError,
Msg: err.Error(),
PayLoad: map[string]interface{}{
PayLoad: map[string]any{
"input": request.Input,
},
})
return
}
fmt.Printf("isFuzzy:%v\n", isFuzzy)
var finalOffset int
if isFuzzy {
var maxOffset int
for index, recommend := range recommends {
offset := model.GetLongestCommonPrefixLength(request.Input, recommend)
if index == 0 || offset > maxOffset {
maxOffset = offset
}
}
finalOffset = maxOffset
} else {
var minOffset int
for index, recommend := range recommends {
offset := model.GetLongestCommonPrefixLength(request.Input, recommend)
@ -44,14 +60,34 @@ func MeasurementRecommendHandler(c *gin.Context) {
minOffset = offset
}
}
finalOffset = minOffset
}
fmt.Printf("finalOffset:%v\n", finalOffset)
resultRecommends := make([]string, 0, len(recommends))
seen := make(map[string]struct{}) // 使用空结构体节省内存
for _, recommend := range recommends {
recommendation := recommend[finalOffset:]
fmt.Printf("resultRecommend:%s\n", recommendation)
fmt.Printf("len of resultRecommend:%d\n", len(recommendation))
if len(recommendation) != 0 {
if _, exists := seen[recommendation]; !exists {
seen[recommendation] = struct{}{}
resultRecommends = append(resultRecommends, recommendation)
}
}
}
c.JSON(http.StatusOK, network.SuccessResponse{
Code: http.StatusOK,
Msg: "success",
PayLoad: map[string]interface{}{
PayLoad: map[string]any{
"input": request.Input,
"offset": minOffset,
"recommended_list": recommends,
"offset": finalOffset,
"recommended_list": resultRecommends,
},
})
}

107
model/redis_recommend.go
View File

@ -23,7 +23,7 @@ func InitAutocompleterWithPool(pool *redigo.Pool) {
}
// RedisSearchRecommend define func of redis search by input string and return recommend results
func RedisSearchRecommend(ctx context.Context, input string) ([]string, error) {
func RedisSearchRecommend(ctx context.Context, input string) ([]string, bool, error) {
rdb := diagram.GetRedisClientInstance()
if input == "" {
@ -33,19 +33,23 @@ func RedisSearchRecommend(ctx context.Context, input string) ([]string, error) {
inputs := strings.Split(input, ".")
inputLen := len(inputs)
fmt.Printf("inputLen:%d\n", inputLen)
switch inputLen {
case 1:
// TODO 优化成NewSet的形式
gridExist, err := rdb.SIsMember(ctx, constants.RedisAllGridSetKey, input).Result()
if err != nil {
logger.Error(ctx, "check grid key exist failed ", "grid_key", input, "error", err)
return []string{}, err
return []string{}, false, err
}
// TODO delete debug info
fmt.Println("gridExist", gridExist)
if !gridExist {
results, err := ac.SuggestOpts(input, redisearch.SuggestOptions{
// TODO 测试如何返回全部的可能模糊结果
searchInput := input
inputLen := len(searchInput)
for inputLen != 0 && !gridExist {
results, err := ac.SuggestOpts(searchInput, redisearch.SuggestOptions{
Num: math.MaxInt16,
Fuzzy: true,
WithScores: false,
@ -56,12 +60,16 @@ func RedisSearchRecommend(ctx context.Context, input string) ([]string, error) {
fmt.Println("err", err)
if err != nil {
logger.Error(ctx, "query info by fuzzy failed", "query_key", input, "error", err)
return []string{}, err
return []string{}, false, err
}
if len(results) == 0 {
// TODO 构建 for 循环返回所有可能的补全
// continue
searchInput = input[:inputLen-1]
inputLen = len(searchInput)
fmt.Printf("next search input:%s\n", searchInput)
fmt.Printf("next search input len:%d\n", inputLen)
continue
}
var grids []string
@ -69,12 +77,12 @@ func RedisSearchRecommend(ctx context.Context, input string) ([]string, error) {
grids = append(grids, result.Term)
}
// 返回模糊查询结果
return grids, nil
return grids, true, nil
}
// 处理 input 不为空、不含有.并且 input 是一个完整的 grid key 的情况
if strings.HasSuffix(input, ".") == false {
return []string{"."}, nil
return []string{"."}, false, nil
}
// 处理 input 不为空并且以.结尾的情况
@ -83,73 +91,85 @@ func RedisSearchRecommend(ctx context.Context, input string) ([]string, error) {
return getSpecificZoneKeys(ctx, setKey)
}
default:
lastToken := inputs[inputLen-1]
lastInput := inputs[inputLen-1]
// 判断 queryKey 是否是空值，空值则返回上一级别下的所有key
if lastToken == "" {
setKey := getConstantsKeyByLength(inputLen - 1)
if lastInput == "" {
fmt.Println("last token is empty")
// TODO 根据所在层级拼接相关setKey
setKey := getCombinedConstantsKeyByLength(inputs[inputLen-2], inputLen)
fmt.Printf("default case set key:%s\n", setKey)
targetSet := diagram.NewRedisSet(ctx, setKey, 10, true)
results, err := targetSet.SMembers(setKey)
if err != nil {
logger.Error(ctx, "get all recommend key by setKey failed", "set_key", setKey, "error", err)
return []string{}, fmt.Errorf("get all recommend key by setKey failed,%w", err)
return []string{}, false, fmt.Errorf("get all recommend key by setKey failed,%w", err)
}
return results, nil
return results, false, nil
}
querykey := lastToken
setKey := getConstantsKeyByLength(inputLen)
setKey := getCombinedConstantsKeyByLength(inputs[inputLen-2], inputLen)
fmt.Printf("default case set key:%s\n", setKey)
targetSet := diagram.NewRedisSet(ctx, setKey, 10, true)
exist, err := targetSet.SIsMember(setKey, querykey)
exist, err := targetSet.SIsMember(setKey, lastInput)
if err != nil {
logger.Error(ctx, "check keys exist failed", "set_key", setKey, "query_key", querykey, "error", err)
return []string{}, fmt.Errorf("check keys failed,%w", err)
logger.Error(ctx, "check keys exist failed", "set_key", setKey, "query_key", lastInput, "error", err)
return []string{}, false, fmt.Errorf("check keys failed,%w", err)
}
if !exist {
searchInput := input
inputLen := len(searchInput)
for inputLen != 0 && !exist {
logger.Info(ctx, "use fuzzy query", "input", input)
results, err := ac.SuggestOpts(input, redisearch.SuggestOptions{
// TODO 测试如何返回全部的可能模糊结果
results, err := ac.SuggestOpts(searchInput, redisearch.SuggestOptions{
Num: math.MaxInt16,
Fuzzy: true,
WithScores: true,
WithPayloads: true,
WithScores: false,
WithPayloads: false,
})
if err != nil {
logger.Error(ctx, "query info by fuzzy failed", "query_key", input, "error", err)
return []string{}, err
return []string{}, false, err
}
if len(results) == 0 {
// TODO 构建 for 循环返回所有可能的补全
searchInput = input[:inputLen-1]
inputLen = len(searchInput)
fmt.Printf("next search input:%s\n", searchInput)
fmt.Printf("next search input len:%d\n", inputLen)
continue
}
fmt.Printf("default fuzzy search results:%v\n", results)
var terms []string
for _, result := range results {
terms = append(terms, result.Term)
}
// 返回模糊查询结果
return terms, nil
return terms, true, nil
}
return []string{}, nil
return []string{input}, false, nil
}
return []string{}, nil
return []string{}, false, nil
}
func getAllGridKeys(ctx context.Context, setKey string) ([]string, error) {
func getAllGridKeys(ctx context.Context, setKey string) ([]string, bool, error) {
// 从redis set 中获取所有的 grid key
gridSets := diagram.NewRedisSet(ctx, setKey, 10, true)
keys, err := gridSets.SMembers("grid_keys")
if err != nil {
return []string{}, fmt.Errorf("get all root keys failed, error: %v", err)
return []string{}, false, fmt.Errorf("get all root keys failed, error: %v", err)
}
return keys, nil
return keys, false, nil
}
func getSpecificZoneKeys(ctx context.Context, setKey string) ([]string, error) {
func getSpecificZoneKeys(ctx context.Context, setKey string) ([]string, bool, error) {
// TODO 从redis set 中获取指定 grid 下的 zone key
zoneSets := diagram.NewRedisSet(ctx, setKey, 10, true)
keys, err := zoneSets.SMembers("grid_keys")
keys, err := zoneSets.SMembers(setKey)
if err != nil {
return []string{}, fmt.Errorf("get all root keys failed, error: %v", err)
return []string{}, false, fmt.Errorf("get all root keys failed, error: %v", err)
}
return keys, nil
return keys, false, nil
}
func getConstantsKeyByLength(inputLen int) string {
@ -167,6 +187,19 @@ func getConstantsKeyByLength(inputLen int) string {
}
}
func getCombinedConstantsKeyByLength(key string, inputLen int) string {
switch inputLen {
case 2:
return fmt.Sprintf(constants.RedisSpecGridZoneSetKey, key)
case 3:
return fmt.Sprintf(constants.RedisSpecZoneStationSetKey, key)
case 4:
return fmt.Sprintf(constants.RedisSpecStationComponentSetKey, key)
default:
return constants.RedisAllGridSetKey
}
}
// GetLongestCommonPrefixLength define func of get longest common prefix length between two strings
func GetLongestCommonPrefixLength(input string, recommendResult string) int {
if input == "" {