feat: synchronize data object attributes with redis

- read parameter and measurement attributes from redis
- support size-aware realtime measurement windows
- coordinate postgreSQL and redis updates with rollback compensation
- validate measurement window sizes during initialization
- add redis change and attribute query/update tests
This commit is contained in:
douxu 2026-07-30 15:32:22 +08:00
parent de1110905e
commit 0b004471fe
12 changed files with 1048 additions and 386 deletions

View File

@ -59,6 +59,13 @@ func validateMeasurementInitializationRecords(records []model.MeasurementInitial
if _, err := model.MeasurementTypeString(record.MeasurementType); err != nil { if _, err := model.MeasurementTypeString(record.MeasurementType); err != nil {
return fmt.Errorf("measurement %q: %w", record.MeasurementTag, err) return fmt.Errorf("measurement %q: %w", record.MeasurementTag, err)
} }
if record.MeasurementSize <= 0 {
return fmt.Errorf(
"measurement %q window size must be greater than 0, got %d",
record.MeasurementTag,
record.MeasurementSize,
)
}
if record.MeasurementDataSource == nil { if record.MeasurementDataSource == nil {
return fmt.Errorf("measurement %q has null data_source", record.MeasurementTag) return fmt.Errorf("measurement %q has null data_source", record.MeasurementTag)
} }

View File

@ -90,6 +90,12 @@ func TestValidateMeasurementInitializationRecords(t *testing.T) {
err = validateMeasurementInitializationRecords([]model.MeasurementInitializationRecord{invalidType}) err = validateMeasurementInitializationRecords([]model.MeasurementInitializationRecord{invalidType})
require.Error(t, err) require.Error(t, err)
assert.Contains(t, err.Error(), "unsupported measurement type -1") assert.Contains(t, err.Error(), "unsupported measurement type -1")
invalidSize := record
invalidSize.MeasurementSize = 0
err = validateMeasurementInitializationRecords([]model.MeasurementInitializationRecord{invalidSize})
require.Error(t, err)
assert.Contains(t, err.Error(), "window size must be greater than 0")
} }
func measurementInitializationRows() *sqlmock.Rows { func measurementInitializationRows() *sqlmock.Rows {
@ -124,6 +130,7 @@ func validMeasurementInitializationRecord() model.MeasurementInitializationRecor
MeasurementTag: "measurement", MeasurementTag: "measurement",
MeasurementType: 0, MeasurementType: 0,
MeasurementMode: 1, MeasurementMode: 1,
MeasurementSize: 1,
MeasurementDataSource: map[string]any{}, MeasurementDataSource: map[string]any{},
MeasurementEventPlan: map[string]any{}, MeasurementEventPlan: map[string]any{},
MeasurementBinding: map[string]any{}, MeasurementBinding: map[string]any{},

View File

@ -4,6 +4,7 @@ package diagram
import ( import (
"context" "context"
"fmt" "fmt"
"sort"
"strconv" "strconv"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
@ -18,40 +19,73 @@ type RedisClient struct {
// greatest numeric timestamp. Measurement ZSets currently store timestamp in // greatest numeric timestamp. Measurement ZSets currently store timestamp in
// member and measurement value in score. // member and measurement value in score.
func (rc *RedisClient) QueryLatestMeasurementValue(ctx context.Context, key string) (float64, error) { func (rc *RedisClient) QueryLatestMeasurementValue(ctx context.Context, key string) (float64, error) {
values, err := rc.QueryLatestMeasurementValues(ctx, key, 1)
if err != nil {
return 0, err
}
return values[0], nil
}
// QueryLatestMeasurementValues returns up to size scores ordered by their
// numeric member timestamps from newest to oldest.
func (rc *RedisClient) QueryLatestMeasurementValues(ctx context.Context, key string, size int) ([]float64, error) {
if rc.Client == nil { if rc.Client == nil {
return 0, fmt.Errorf("redis client is not initialized") return nil, fmt.Errorf("redis client is not initialized")
}
if size <= 0 {
return nil, fmt.Errorf("measurement window size must be greater than 0, got %d", size)
} }
members, err := rc.Client.ZRangeWithScores(ctx, key, 0, -1).Result() members, err := rc.Client.ZRangeWithScores(ctx, key, 0, -1).Result()
if err != nil { if err != nil {
return 0, err return nil, err
} }
return latestMeasurementValue(members, key) return latestMeasurementValues(members, key, size)
} }
func latestMeasurementValue(members []redis.Z, key string) (float64, error) { func latestMeasurementValue(members []redis.Z, key string) (float64, error) {
if len(members) == 0 { values, err := latestMeasurementValues(members, key, 1)
return 0, fmt.Errorf("real-time measurement value not found for key %q", key) if err != nil {
return 0, err
}
return values[0], nil
} }
var latestTimestamp int64 func latestMeasurementValues(members []redis.Z, key string, size int) ([]float64, error) {
var latestValue float64 if size <= 0 {
found := false return nil, fmt.Errorf("measurement window size must be greater than 0, got %d", size)
}
if len(members) == 0 {
return nil, fmt.Errorf("real-time measurement value not found for key %q", key)
}
type timestampedValue struct {
timestamp int64
value float64
}
values := make([]timestampedValue, 0, len(members))
for _, member := range members { for _, member := range members {
timestamp, err := strconv.ParseInt(fmt.Sprint(member.Member), 10, 64) timestamp, err := strconv.ParseInt(fmt.Sprint(member.Member), 10, 64)
if err != nil { if err != nil {
continue continue
} }
if !found || timestamp > latestTimestamp { values = append(values, timestampedValue{timestamp: timestamp, value: member.Score})
latestTimestamp = timestamp
latestValue = member.Score
found = true
} }
if len(values) == 0 {
return nil, fmt.Errorf("real-time measurement timestamps are invalid for key %q", key)
} }
if !found {
return 0, fmt.Errorf("real-time measurement timestamps are invalid for key %q", key) sort.Slice(values, func(i, j int) bool {
return values[i].timestamp > values[j].timestamp
})
if size > len(values) {
size = len(values)
} }
return latestValue, nil result := make([]float64, size)
for index := range size {
result[index] = values[index].value
}
return result, nil
} }
// NewRedisClient define func of new redis client instance // NewRedisClient define func of new redis client instance

View File

@ -26,3 +26,32 @@ func TestLatestMeasurementValueRejectsMissingOrInvalidTimestamps(t *testing.T) {
_, err = latestMeasurementValue([]redis.Z{{Member: "invalid", Score: 1}}, "measurement-key") _, err = latestMeasurementValue([]redis.Z{{Member: "invalid", Score: 1}}, "measurement-key")
require.Error(t, err) require.Error(t, err)
} }
func TestLatestMeasurementValuesReturnsNewestWindow(t *testing.T) {
values, err := latestMeasurementValues([]redis.Z{
{Member: "100", Score: 10},
{Member: "400", Score: 40},
{Member: "invalid", Score: 999},
{Member: "200", Score: 20},
{Member: "300", Score: 30},
}, "measurement-key", 3)
require.NoError(t, err)
assert.Equal(t, []float64{40, 30, 20}, values)
}
func TestLatestMeasurementValuesReturnsAvailableWindow(t *testing.T) {
values, err := latestMeasurementValues([]redis.Z{
{Member: "100", Score: 10},
{Member: "200", Score: 20},
}, "measurement-key", 5)
require.NoError(t, err)
assert.Equal(t, []float64{20, 10}, values)
}
func TestLatestMeasurementValuesRejectsInvalidSize(t *testing.T) {
_, err := latestMeasurementValues([]redis.Z{{Member: "100", Score: 10}}, "measurement-key", 0)
require.Error(t, err)
assert.Contains(t, err.Error(), "window size must be greater than 0")
}

View File

@ -3,26 +3,27 @@ package handler
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"strconv"
"strings" "strings"
"modelRT/common" "modelRT/common"
"modelRT/common/errcode" "modelRT/common/errcode"
"modelRT/constants" "modelRT/constants"
"modelRT/database"
"modelRT/diagram" "modelRT/diagram"
"modelRT/logger" "modelRT/logger"
"modelRT/model" "modelRT/model"
"modelRT/orm" "modelRT/orm"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/redis/go-redis/v9"
) )
// DataObjectAttributeQueryHandler define data object attribute value query process API // DataObjectAttributeQueryHandler define data object attribute value query process API
func DataObjectAttributeQueryHandler(c *gin.Context) { func DataObjectAttributeQueryHandler(c *gin.Context) {
ctx := c.Request.Context() ctx := c.Request.Context()
pgClient := database.GetPostgresDBClient()
token, field, err := parseDataObjectAttributeQuery(c) token, field, err := parseDataObjectAttributeQuery(c)
if err != nil { if err != nil {
@ -44,96 +45,24 @@ func DataObjectAttributeQueryHandler(c *gin.Context) {
return return
} }
var parameter *database.ParameterDataObject value, err := queryDataObjectAttributeValue(
var measurement *orm.Measurement
var measurementComponent *orm.Component
switch dataObjectType {
case constants.DataObjectTypeParameter:
// 参量支持两种形式token4.token5.token6.token7与token1.token2.token3.token4.token5.token6.token7
parameter, err = database.QueryParameterByDataObjectToken(ctx, pgClient, token)
if err != nil {
if errors.Is(err, common.ErrInvalidParameterToken) ||
errors.Is(err, common.ErrParameterTokenNotFound) ||
errors.Is(err, common.ErrAmbiguousParameterToken) {
logger.Warn(ctx, "validate parameter token failed", "token", token, "error", err)
renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil)
return
}
logger.Error(ctx, "query parameter token from postgres failed", "token", token, "error", err)
renderRespFailure(c, constants.RespCodeServerError, "validate parameter token failed", nil)
return
}
case constants.DataObjectTypeMeasurement:
// 量测支持token1.token2.token3.token4.token5.token6.token7、token4.token5.token6.token7、token4.token7
measurement, measurementComponent, err = database.QueryMeasurementByDataObjectToken(ctx, pgClient, token)
if err != nil {
if errors.Is(err, common.ErrInvalidMeasurementToken) ||
errors.Is(err, common.ErrMeasurementTokenNotFound) ||
errors.Is(err, common.ErrAmbiguousMeasurementToken) {
logger.Warn(ctx, "validate measurement token failed", "token", token, "error", err)
renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil)
return
}
logger.Error(ctx, "query measurement token from postgres failed", "token", token, "error", err)
renderRespFailure(c, constants.RespCodeServerError, "validate measurement token failed", nil)
return
}
}
switch dataObjectType {
case constants.DataObjectTypeParameter:
value, err := buildParameterAttributeValue(
ctx, ctx,
dataObjectType,
token,
field, field,
parameter, loadDataObjectHashField,
func(ctx context.Context, parameter *database.ParameterDataObject) (any, error) { loadMeasurementValueMetadata,
return database.QueryParameterDataObjectValue(ctx, pgClient, parameter)
},
func(ctx context.Context, attributeName string) (string, error) {
return database.QueryParameterAttributeDescription(ctx, pgClient, attributeName)
},
)
if err != nil {
if errors.Is(err, common.ErrUnsupportedParameterField) {
logger.Warn(ctx, "query unsupported parameter field", "token", token, "field", field, "error", err)
renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil)
return
}
logger.Error(ctx, "build parameter attribute value failed", "token", token, "field", field, "error", err)
renderRespFailure(c, constants.RespCodeServerError, "query parameter attribute failed", nil)
return
}
result := dataObjectAttributeQueryResult{
Token: token,
Field: field,
Code: errcode.ErrProcessSuccess.Code(),
Msg: errcode.ErrProcessSuccess.Msg(),
Value: value,
}
renderRespSuccess(c, constants.RespCodeSuccess, "query parameter attribute success", map[string]any{
"attributes": []dataObjectAttributeQueryResult{result},
})
case constants.DataObjectTypeMeasurement:
value, err := buildMeasurementAttributeValue(
ctx,
field,
measurement,
measurementComponent,
queryMeasurementRealtimeValue, queryMeasurementRealtimeValue,
) )
if err != nil { if err != nil {
if errors.Is(err, common.ErrUnsupportedMeasurementField) { if isDataObjectTokenNotFound(err) {
logger.Warn(ctx, "query unsupported measurement field", "token", token, "field", field, "error", err) logger.Warn(ctx, "query data-object token from redis failed", "token", token, "field", field, "error", err)
renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil) renderRespFailure(c, constants.RespCodeInvalidParams, err.Error(), nil)
return return
} }
logger.Error(ctx, "build measurement attribute value failed", "token", token, "field", field, "error", err) logger.Error(ctx, "query data-object attribute from redis failed", "token", token, "field", field, "error", err)
renderRespFailure(c, constants.RespCodeServerError, "query measurement attribute failed", nil) renderRespFailure(c, constants.RespCodeServerError, dataObjectAttributeFailureMessage(dataObjectType), nil)
return return
} }
@ -144,12 +73,9 @@ func DataObjectAttributeQueryHandler(c *gin.Context) {
Msg: errcode.ErrProcessSuccess.Msg(), Msg: errcode.ErrProcessSuccess.Msg(),
Value: value, Value: value,
} }
renderRespSuccess(c, constants.RespCodeSuccess, "query measurement attribute success", map[string]any{ renderRespSuccess(c, constants.RespCodeSuccess, dataObjectAttributeSuccessMessage(dataObjectType), map[string]any{
"attributes": []dataObjectAttributeQueryResult{result}, "attributes": []dataObjectAttributeQueryResult{result},
}) })
default:
renderRespFailure(c, constants.RespCodeInvalidParams, "invalid data object type", nil)
}
} }
func parseDataObjectAttributeQuery(c *gin.Context) (string, string, error) { func parseDataObjectAttributeQuery(c *gin.Context) (string, string, error) {
@ -165,11 +91,11 @@ func parseDataObjectAttributeQuery(c *gin.Context) (string, string, error) {
return token, field, nil return token, field, nil
} }
type measurementValueLoader func(context.Context, orm.JSONMap) (any, error) type dataObjectHashFieldLoader func(context.Context, constants.DataObjectType, string, string) (string, error)
type parameterValueLoader func(context.Context, *database.ParameterDataObject) (any, error) type measurementValueMetadataLoader func(context.Context, string) (orm.JSONMap, int, error)
type parameterDescriptionLoader func(context.Context, string) (string, error) type measurementValueLoader func(context.Context, orm.JSONMap, int) (any, error)
var measurementDataObjectFields = map[string]struct{}{ var measurementDataObjectFields = map[string]struct{}{
"value": {}, "value": {},
@ -220,116 +146,242 @@ func validateDataObjectField(dataObjectType constants.DataObjectType, field stri
} }
} }
func buildParameterAttributeValue( func queryDataObjectAttributeValue(
ctx context.Context, ctx context.Context,
dataObjectType constants.DataObjectType,
token string,
field string, field string,
parameter *database.ParameterDataObject, loadHashField dataObjectHashFieldLoader,
loadValue parameterValueLoader, loadMeasurementMetadata measurementValueMetadataLoader,
loadDescription parameterDescriptionLoader, loadMeasurementValue measurementValueLoader,
) (any, error) { ) (any, error) {
if parameter == nil { if dataObjectType == constants.DataObjectTypeMeasurement && field == "value" {
return nil, fmt.Errorf("parameter data object is nil") if loadMeasurementMetadata == nil {
return nil, fmt.Errorf("measurement value metadata loader is nil")
} }
if loadMeasurementValue == nil {
component := parameter.Component
switch field {
case "value":
if loadValue == nil {
return nil, fmt.Errorf("parameter value loader is nil")
}
return loadValue(ctx, parameter)
case "meta":
return "PARAM", nil
case "type":
return parameter.AttributeType, nil
case "name":
return strings.Join([]string{
component.NSPath,
component.Tag,
parameter.AttributeGroup,
parameter.AttributeName,
}, "."), nil
case "description":
if loadDescription == nil {
return nil, fmt.Errorf("parameter description loader is nil")
}
return loadDescription(ctx, parameter.AttributeName)
case "id":
return strings.Join([]string{
component.GridName,
component.ZoneName,
component.StationName,
component.NSPath,
component.Tag,
parameter.AttributeGroup,
parameter.AttributeName,
}, "."), nil
default:
return nil, fmt.Errorf("%w: %s", common.ErrUnsupportedParameterField, field)
}
}
func buildMeasurementAttributeValue(
ctx context.Context,
field string,
measurement *orm.Measurement,
component *orm.Component,
loadValue measurementValueLoader,
) (any, error) {
if measurement == nil {
return nil, fmt.Errorf("measurement is nil")
}
if component == nil {
return nil, fmt.Errorf("measurement component is nil")
}
switch field {
case "value":
if loadValue == nil {
return nil, fmt.Errorf("measurement value loader is nil") return nil, fmt.Errorf("measurement value loader is nil")
} }
return loadValue(ctx, measurement.DataSource) dataSource, size, err := loadMeasurementMetadata(ctx, token)
case "mode": if err != nil {
return measurement.Mode, nil return nil, err
case "meta": }
return "MEASUREMENT", nil return loadMeasurementValue(ctx, dataSource, size)
case "type": }
return model.MeasurementTypeString(measurement.Type)
case "name": if loadHashField == nil {
// The resolved measurement and component prove that token4.token7 exists. return nil, fmt.Errorf("data-object hash field loader is nil")
return component.NSPath + "." + measurement.Tag, nil }
case "description": rawValue, err := loadHashField(ctx, dataObjectType, token, field)
return measurement.Name, nil if err != nil {
case "id": return nil, err
return strings.Join([]string{ }
component.GridName, if dataObjectType == constants.DataObjectTypeParameter && field == "value" {
component.ZoneName, attributeType, err := loadHashField(ctx, dataObjectType, token, "type")
component.StationName, if err != nil {
component.NSPath, return nil, err
component.Tag, }
"bay", return decodeParameterHashValue(rawValue, attributeType)
measurement.Tag, }
}, "."), nil return decodeDataObjectHashField(dataObjectType, field, rawValue)
case "size": }
return measurement.Size, nil
case "data_source": func loadDataObjectHashField(
return measurement.DataSource, nil ctx context.Context,
case "event_plan": dataObjectType constants.DataObjectType,
return measurement.EventPlan, nil token string,
case "binding": field string,
return measurement.Binding, nil ) (string, error) {
rdb := diagram.GetRedisClientInstance()
if rdb == nil {
return "", fmt.Errorf("redis client is not initialized")
}
value, err := rdb.HGet(ctx, token, field).Result()
if errors.Is(err, redis.Nil) {
exists, existsErr := rdb.Exists(ctx, token).Result()
if existsErr != nil {
return "", fmt.Errorf("check redis data-object hash %q: %w", token, existsErr)
}
if exists > 0 {
return "", fmt.Errorf("redis data-object hash %q does not contain field %q", token, field)
}
switch dataObjectType {
case constants.DataObjectTypeParameter:
return "", fmt.Errorf("%w: %q", common.ErrParameterTokenNotFound, token)
case constants.DataObjectTypeMeasurement:
return "", fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
default: default:
return nil, fmt.Errorf("%w: %s", common.ErrUnsupportedMeasurementField, field) return "", fmt.Errorf("invalid data object type %q", dataObjectType)
}
}
if err != nil {
return "", fmt.Errorf("query redis hash %q field %q: %w", token, field, err)
}
return value, nil
}
func loadMeasurementValueMetadata(ctx context.Context, token string) (orm.JSONMap, int, error) {
rdb := diagram.GetRedisClientInstance()
if rdb == nil {
return nil, 0, fmt.Errorf("redis client is not initialized")
}
values, err := rdb.HMGet(ctx, token, "data_source", "size").Result()
if err != nil {
return nil, 0, fmt.Errorf("query redis hash %q measurement value metadata: %w", token, err)
}
if len(values) != 2 {
return nil, 0, fmt.Errorf("redis hash %q returned %d measurement metadata fields", token, len(values))
}
if values[0] == nil || values[1] == nil {
exists, existsErr := rdb.Exists(ctx, token).Result()
if existsErr != nil {
return nil, 0, fmt.Errorf("check redis data-object hash %q: %w", token, existsErr)
}
if exists == 0 {
return nil, 0, fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
}
missingFields := make([]string, 0, 2)
if values[0] == nil {
missingFields = append(missingFields, "data_source")
}
if values[1] == nil {
missingFields = append(missingFields, "size")
}
return nil, 0, fmt.Errorf(
"redis measurement hash %q does not contain field(s) %s",
token,
strings.Join(missingFields, ", "),
)
}
rawDataSource, ok := values[0].(string)
if !ok {
return nil, 0, fmt.Errorf("redis measurement hash %q data_source has type %T", token, values[0])
}
var dataSource orm.JSONMap
if err := json.Unmarshal([]byte(rawDataSource), &dataSource); err != nil {
return nil, 0, fmt.Errorf("decode measurement data_source from redis hash %q: %w", token, err)
}
rawSize, ok := values[1].(string)
if !ok {
return nil, 0, fmt.Errorf("redis measurement hash %q size has type %T", token, values[1])
}
size, err := strconv.Atoi(rawSize)
if err != nil {
return nil, 0, fmt.Errorf("decode measurement size %q: %w", rawSize, err)
}
if size <= 0 {
return nil, 0, fmt.Errorf("measurement window size must be greater than 0, got %d", size)
}
return dataSource, size, nil
}
func decodeDataObjectHashField(
dataObjectType constants.DataObjectType,
field string,
rawValue string,
) (any, error) {
if dataObjectType != constants.DataObjectTypeMeasurement {
return rawValue, nil
}
switch field {
case "mode":
value, err := strconv.ParseInt(rawValue, 10, 16)
if err != nil {
return nil, fmt.Errorf("decode measurement mode %q: %w", rawValue, err)
}
return int16(value), nil
case "size":
value, err := strconv.Atoi(rawValue)
if err != nil {
return nil, fmt.Errorf("decode measurement size %q: %w", rawValue, err)
}
return value, nil
case "data_source", "event_plan", "binding":
return decodeRedisJSON(rawValue)
default:
return rawValue, nil
} }
} }
func queryMeasurementRealtimeValue(ctx context.Context, dataSource orm.JSONMap) (any, error) { func decodeParameterHashValue(rawValue, attributeType string) (any, error) {
normalizedType := strings.ToUpper(strings.TrimSpace(attributeType))
if rawValue == "null" {
return nil, nil
}
switch {
case normalizedType == "BOOLEAN":
value, err := strconv.ParseBool(rawValue)
if err != nil {
return nil, fmt.Errorf("decode parameter boolean value %q: %w", rawValue, err)
}
return value, nil
case normalizedType == "SMALLINT",
normalizedType == "INTEGER",
normalizedType == "BIGINT":
value, err := strconv.ParseInt(rawValue, 10, 64)
if err != nil {
return nil, fmt.Errorf("decode parameter integer value %q: %w", rawValue, err)
}
return value, nil
case normalizedType == "REAL",
normalizedType == "DOUBLE PRECISION",
strings.HasPrefix(normalizedType, "NUMERIC"),
strings.HasPrefix(normalizedType, "DECIMAL"):
if _, err := strconv.ParseFloat(rawValue, 64); err != nil {
return nil, fmt.Errorf("decode parameter numeric value %q: %w", rawValue, err)
}
return json.Number(rawValue), nil
case normalizedType == "JSON",
normalizedType == "JSONB",
strings.HasSuffix(normalizedType, "[]"):
return decodeRedisJSON(rawValue)
default:
return rawValue, nil
}
}
func decodeRedisJSON(rawValue string) (any, error) {
var value any
decoder := json.NewDecoder(strings.NewReader(rawValue))
decoder.UseNumber()
if err := decoder.Decode(&value); err != nil {
return nil, fmt.Errorf("decode redis JSON value: %w", err)
}
return value, nil
}
func isDataObjectTokenNotFound(err error) bool {
return errors.Is(err, common.ErrParameterTokenNotFound) ||
errors.Is(err, common.ErrMeasurementTokenNotFound)
}
func dataObjectAttributeSuccessMessage(dataObjectType constants.DataObjectType) string {
if dataObjectType == constants.DataObjectTypeParameter {
return "query parameter attribute success"
}
return "query measurement attribute success"
}
func dataObjectAttributeFailureMessage(dataObjectType constants.DataObjectType) string {
if dataObjectType == constants.DataObjectTypeParameter {
return "query parameter attribute failed"
}
return "query measurement attribute failed"
}
func queryMeasurementRealtimeValue(ctx context.Context, dataSource orm.JSONMap, size int) (any, error) {
queryKey, err := model.GenerateMeasureIdentifier(dataSource) queryKey, err := model.GenerateMeasureIdentifier(dataSource)
if err != nil { if err != nil {
return nil, fmt.Errorf("generate measurement redis key: %w", err) return nil, fmt.Errorf("generate measurement redis key: %w", err)
} }
value, err := diagram.NewRedisClient().QueryLatestMeasurementValue(ctx, queryKey) value, err := diagram.NewRedisClient().QueryLatestMeasurementValues(ctx, queryKey, size)
if err != nil { if err != nil {
return nil, fmt.Errorf("query real-time measurement value by key %q: %w", queryKey, err) return nil, fmt.Errorf("query real-time measurement value by key %q: %w", queryKey, err)
} }

View File

@ -2,13 +2,14 @@ package handler
import ( import (
"context" "context"
"encoding/json"
"fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"testing" "testing"
"modelRT/common" "modelRT/common"
"modelRT/constants" "modelRT/constants"
"modelRT/database"
"modelRT/orm" "modelRT/orm"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
@ -117,161 +118,156 @@ func TestValidateDataObjectField(t *testing.T) {
} }
} }
func TestBuildParameterAttributeValue(t *testing.T) { func TestQueryParameterAttributeValueFromRedisHash(t *testing.T) {
parameter := &database.ParameterDataObject{ fields := map[string]string{
Component: orm.Component{ "value": "220.50",
GridName: "grid000", "type": "DOUBLE PRECISION",
ZoneName: "zone000", "name": "110kV_TV.cable_22.rated.rated_voltage",
StationName: "station000", "description": "额定电压",
NSPath: "110kV_TV",
Tag: "cable_22",
},
AttributeGroup: "rated",
AttributeName: "rated_voltage",
AttributeType: "DOUBLE PRECISION",
}
loader := func(_ context.Context, actual *database.ParameterDataObject) (any, error) {
assert.Same(t, parameter, actual)
return float64(220), nil
}
descriptionLoader := func(_ context.Context, attributeName string) (string, error) {
assert.Equal(t, "rated_voltage", attributeName)
return "额定电压", nil
} }
loader := hashFieldLoaderForTest(fields)
tests := []struct { value, err := queryDataObjectAttributeValue(
field string
expected any
}{
{field: "value", expected: float64(220)},
{field: "meta", expected: "PARAM"},
{field: "type", expected: "DOUBLE PRECISION"},
{field: "name", expected: "110kV_TV.cable_22.rated.rated_voltage"},
{field: "description", expected: "额定电压"},
{field: "id", expected: "grid000.zone000.station000.110kV_TV.cable_22.rated.rated_voltage"},
}
for _, tt := range tests {
t.Run(tt.field, func(t *testing.T) {
actual, err := buildParameterAttributeValue(
context.Background(), context.Background(),
tt.field, constants.DataObjectTypeParameter,
parameter, "parameter-token",
"value",
loader, loader,
descriptionLoader, nil,
nil,
) )
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, tt.expected, actual) assert.Equal(t, json.Number("220.50"), value)
})
}
}
func TestBuildParameterAttributeValueRejectsUnsupportedField(t *testing.T) { description, err := queryDataObjectAttributeValue(
_, err := buildParameterAttributeValue(
context.Background(), context.Background(),
"unknown", constants.DataObjectTypeParameter,
&database.ParameterDataObject{}, "parameter-token",
"description",
loader,
nil, nil,
nil, nil,
) )
require.Error(t, err) require.NoError(t, err)
assert.ErrorIs(t, err, common.ErrUnsupportedParameterField) assert.Equal(t, "额定电压", description)
} }
func TestBuildMeasurementAttributeValue(t *testing.T) { func TestQueryMeasurementAttributeValueFromRedisHash(t *testing.T) {
fields := map[string]string{
"mode": "1",
"size": "10",
"name": "110kV_TV.IA_rms",
"data_source": `{"type":1,"io_address":{"channel":"tm1p"}}`,
"event_plan": `{"enabled":true}`,
}
loader := hashFieldLoaderForTest(fields)
mode, err := queryDataObjectAttributeValue(
context.Background(),
constants.DataObjectTypeMeasurement,
"measurement-token",
"mode",
loader,
nil,
nil,
)
require.NoError(t, err)
assert.Equal(t, int16(1), mode)
eventPlan, err := queryDataObjectAttributeValue(
context.Background(),
constants.DataObjectTypeMeasurement,
"measurement-token",
"event_plan",
loader,
nil,
nil,
)
require.NoError(t, err)
assert.Equal(t, map[string]any{"enabled": true}, eventPlan)
}
func TestQueryMeasurementRealtimeValueUsesDataSourceFromRedisHash(t *testing.T) {
dataSource := orm.JSONMap{ dataSource := orm.JSONMap{
"type": float64(1), "type": float64(1),
"io_address": map[string]any{ "io_address": map[string]any{
"station": "001",
"channel": "tm1p", "channel": "tm1p",
}, },
} }
eventPlan := orm.JSONMap{"enabled": true} metadataLoader := func(_ context.Context, token string) (orm.JSONMap, int, error) {
binding := orm.JSONMap{"ct": map[string]any{"ratio": float64(2)}} assert.Equal(t, "measurement-token", token)
measurement := &orm.Measurement{ return dataSource, 2, nil
Tag: "IA_rms",
Name: "A相电流",
Type: 0,
Mode: 1,
Size: 10,
DataSource: dataSource,
EventPlan: eventPlan,
Binding: binding,
} }
component := &orm.Component{ valueLoader := func(_ context.Context, dataSource orm.JSONMap, size int) (any, error) {
GridName: "grid000", assert.Equal(t, float64(1), dataSource["type"])
ZoneName: "zone000", assert.Equal(t, "001", dataSource["io_address"].(map[string]any)["station"])
StationName: "station000", assert.Equal(t, 2, size)
NSPath: "110kV_TV", return []float64{220, 219.5}, nil
Tag: "cable_22",
} }
loader := func(_ context.Context, source orm.JSONMap) (any, error) { value, err := queryDataObjectAttributeValue(
assert.Equal(t, dataSource, source)
return float64(220), nil
}
tests := []struct {
field string
expected any
}{
{field: "value", expected: float64(220)},
{field: "mode", expected: int16(1)},
{field: "meta", expected: "MEASUREMENT"},
{field: "type", expected: "TM"},
{field: "name", expected: "110kV_TV.IA_rms"},
{field: "description", expected: "A相电流"},
{field: "id", expected: "grid000.zone000.station000.110kV_TV.cable_22.bay.IA_rms"},
{field: "size", expected: 10},
{field: "data_source", expected: dataSource},
{field: "event_plan", expected: eventPlan},
{field: "binding", expected: binding},
}
for _, tt := range tests {
t.Run(tt.field, func(t *testing.T) {
actual, err := buildMeasurementAttributeValue(context.Background(), tt.field, measurement, component, loader)
require.NoError(t, err)
assert.Equal(t, tt.expected, actual)
})
}
}
func TestBuildMeasurementAttributeValueRejectsUnsupportedField(t *testing.T) {
_, err := buildMeasurementAttributeValue(
context.Background(), context.Background(),
"unknown", constants.DataObjectTypeMeasurement,
&orm.Measurement{}, "measurement-token",
&orm.Component{}, "value",
nil,
metadataLoader,
valueLoader,
)
require.NoError(t, err)
assert.Equal(t, []float64{220, 219.5}, value)
}
func TestQueryDataObjectAttributeValuePropagatesRedisTokenNotFound(t *testing.T) {
loader := func(context.Context, constants.DataObjectType, string, string) (string, error) {
return "", fmt.Errorf("%w: token", common.ErrParameterTokenNotFound)
}
_, err := queryDataObjectAttributeValue(
context.Background(),
constants.DataObjectTypeParameter,
"missing-token",
"name",
loader,
nil,
nil, nil,
) )
require.Error(t, err) require.Error(t, err)
assert.ErrorIs(t, err, common.ErrUnsupportedMeasurementField) assert.ErrorIs(t, err, common.ErrParameterTokenNotFound)
assert.True(t, isDataObjectTokenNotFound(err))
} }
func TestBuildMeasurementAttributeValueMode(t *testing.T) { func TestDecodeParameterHashValue(t *testing.T) {
component := &orm.Component{}
tests := []struct { tests := []struct {
name string name string
mode int16 rawValue string
expected int16 attributeType string
expected any
}{ }{
{name: "collected value", mode: 1, expected: 1}, {name: "boolean", rawValue: "true", attributeType: "BOOLEAN", expected: true},
{name: "manually assigned value", mode: 0, expected: 0}, {name: "integer", rawValue: "42", attributeType: "INTEGER", expected: int64(42)},
{name: "other positive mode", mode: 2, expected: 2}, {name: "numeric", rawValue: "1234567890.123456789", attributeType: "NUMERIC(30,9)", expected: json.Number("1234567890.123456789")},
{name: "negative mode", mode: -1, expected: -1}, {name: "jsonb", rawValue: `{"key":"value"}`, attributeType: "JSONB", expected: map[string]any{"key": "value"}},
{name: "string", rawValue: "cable", attributeType: "CHARACTER VARYING(64)", expected: "cable"},
{name: "null", rawValue: "null", attributeType: "INTEGER", expected: nil},
} }
for _, tt := range tests { for _, test := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
actual, err := buildMeasurementAttributeValue( actual, err := decodeParameterHashValue(test.rawValue, test.attributeType)
context.Background(),
"mode",
&orm.Measurement{Mode: tt.mode},
component,
nil,
)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, tt.expected, actual) assert.Equal(t, test.expected, actual)
}) })
} }
} }
func hashFieldLoaderForTest(fields map[string]string) dataObjectHashFieldLoader {
return func(_ context.Context, _ constants.DataObjectType, _ string, field string) (string, error) {
value, exists := fields[field]
if !exists {
return "", fmt.Errorf("field %q not found", field)
}
return value, nil
}
}

View File

@ -61,6 +61,7 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
} }
}() }()
redisChanges := NewRedisChangeSet(diagram.GetRedisClientInstance())
message := "data-object attribute update success" message := "data-object attribute update success"
var measurementResult measurementUpdateResult var measurementResult measurementUpdateResult
switch dataObjectType { switch dataObjectType {
@ -69,14 +70,24 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
if queryErr == nil { if queryErr == nil {
queryErr = database.UpdateParameterDataObjectValue(ctx, tx, parameter, value) queryErr = database.UpdateParameterDataObjectValue(ctx, tx, parameter, value)
} }
if queryErr == nil {
queryErr = redisChanges.AddDataObjectHashChange(ctx, dataObjectType, request.Token, field, value)
}
err = queryErr err = queryErr
case constants.DataObjectTypeMeasurement: case constants.DataObjectTypeMeasurement:
measurementResult, err = updateMeasurementDataObject(ctx, tx, request.Token, field, value, request.Data, measurementUpdateDependencies{ measurementResult, err = updateMeasurementDataObject(ctx, tx, request.Token, field, value, request.Data, measurementUpdateDependencies{
writeManualValueFunc: writeMeasurementManualValue, writeManualValueFunc: func(ctx context.Context, measurement *orm.Measurement, value float64) error {
return redisChanges.AddMeasurementValueChange(ctx, measurement, value, false)
},
updateDataRTFunc: callRealTimeDataWriteStopInterface, updateDataRTFunc: callRealTimeDataWriteStopInterface,
startDataRTFunc: callRealTimeDataWriteStartInterface, startDataRTFunc: callRealTimeDataWriteStartInterface,
replaceRedisValueFunc: replaceMeasurementRedisValue, replaceRedisValueFunc: func(ctx context.Context, measurement *orm.Measurement, value float64) error {
return redisChanges.AddMeasurementValueChange(ctx, measurement, value, true)
},
}) })
if err == nil && measurementResult.modeChanged {
err = redisChanges.AddDataObjectHashChange(ctx, dataObjectType, request.Token, "mode", measurementResult.mode)
}
message = measurementResult.message message = measurementResult.message
default: default:
err = fmt.Errorf("unsupported data object type %q", dataObjectType) err = fmt.Errorf("unsupported data object type %q", dataObjectType)
@ -84,7 +95,7 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
if err != nil { if err != nil {
_ = tx.Rollback().Error _ = tx.Rollback().Error
if measurementResult.recordFailure { if measurementResult.recordFailureOnError {
if logErr := database.AppendMeasurementValueOperation(ctx, database.GetPostgresDBClient(), measurementResult.measurementID, 1, measurementResult.value, time.Now().UTC()); logErr != nil { if logErr := database.AppendMeasurementValueOperation(ctx, database.GetPostgresDBClient(), measurementResult.measurementID, 1, measurementResult.value, time.Now().UTC()); logErr != nil {
logger.Error(ctx, "append failed measurement value operation failed", "measurement_id", measurementResult.measurementID, "error", logErr) logger.Error(ctx, "append failed measurement value operation failed", "measurement_id", measurementResult.measurementID, "error", logErr)
} }
@ -98,7 +109,29 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
return return
} }
if err := redisChanges.Apply(ctx); err != nil {
_ = tx.Rollback().Error
if measurementResult.recordFailureOnError {
if logErr := database.AppendMeasurementValueOperation(ctx, database.GetPostgresDBClient(), measurementResult.measurementID, 1, measurementResult.value, time.Now().UTC()); logErr != nil {
logger.Error(ctx, "append failed measurement value operation failed", "measurement_id", measurementResult.measurementID, "error", logErr)
}
}
logger.Error(ctx, "apply redis data-object changes failed", "token", request.Token, "field", field, "error", err)
renderRespFailure(c, constants.RespCodeFailed, err.Error(), nil)
return
}
if err := tx.Commit().Error; err != nil { if err := tx.Commit().Error; err != nil {
revertCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), redisChangeRestoreTimeout)
defer cancel()
if redisErr := redisChanges.Revert(revertCtx); redisErr != nil {
logger.Error(ctx, "revert redis data-object changes failed", "token", request.Token, "field", field, "error", redisErr)
}
if measurementResult.recordFailureOnError {
if logErr := database.AppendMeasurementValueOperation(ctx, database.GetPostgresDBClient(), measurementResult.measurementID, 1, measurementResult.value, time.Now().UTC()); logErr != nil {
logger.Error(ctx, "append failed measurement value operation failed", "measurement_id", measurementResult.measurementID, "error", logErr)
}
}
logger.Error(ctx, "commit data-object update transaction failed", "token", request.Token, "field", field, "error", err) logger.Error(ctx, "commit data-object update transaction failed", "token", request.Token, "field", field, "error", err)
renderRespFailure(c, constants.RespCodeServerError, "transaction commit failed", nil) renderRespFailure(c, constants.RespCodeServerError, "transaction commit failed", nil)
return return
@ -236,7 +269,9 @@ type measurementUpdateResult struct {
message string message string
measurementID int64 measurementID int64
value float64 value float64
recordFailure bool recordFailureOnError bool
mode int16
modeChanged bool
} }
func updateMeasurementDataObject( func updateMeasurementDataObject(
@ -305,7 +340,11 @@ func updateMeasurementDataObject(
return measurementUpdateResult{}, fmt.Errorf("start automatic measurement write to dataRT: %w", err) return measurementUpdateResult{}, fmt.Errorf("start automatic measurement write to dataRT: %w", err)
} }
} }
return measurementUpdateResult{message: fmt.Sprintf("measurement mode changed to %s", measurementModeName(mode))}, nil return measurementUpdateResult{
message: fmt.Sprintf("measurement mode changed to %s", measurementModeName(mode)),
mode: mode,
modeChanged: true,
}, nil
case "value": case "value":
currentMode, err := measurementModeIsAutomatic(lockedMeasurement.Mode) currentMode, err := measurementModeIsAutomatic(lockedMeasurement.Mode)
if err != nil { if err != nil {
@ -321,7 +360,7 @@ func updateMeasurementDataObject(
failureResult := measurementUpdateResult{ failureResult := measurementUpdateResult{
measurementID: lockedMeasurement.ID, measurementID: lockedMeasurement.ID,
value: manualValue, value: manualValue,
recordFailure: true, recordFailureOnError: true,
} }
if dependencies.writeManualValueFunc == nil { if dependencies.writeManualValueFunc == nil {
return failureResult, errcode.ErrMeasurementValueUpdateFailed.WithCause(fmt.Errorf("measurement manual value writer is nil")) return failureResult, errcode.ErrMeasurementValueUpdateFailed.WithCause(fmt.Errorf("measurement manual value writer is nil"))
@ -338,7 +377,8 @@ func updateMeasurementDataObject(
if err := database.AppendMeasurementValueOperation(ctx, tx, lockedMeasurement.ID, 0, manualValue, time.Now().UTC()); err != nil { if err := database.AppendMeasurementValueOperation(ctx, tx, lockedMeasurement.ID, 0, manualValue, time.Now().UTC()); err != nil {
return failureResult, errcode.ErrMeasurementValueUpdateFailed.WithCause(err) return failureResult, errcode.ErrMeasurementValueUpdateFailed.WithCause(err)
} }
return measurementUpdateResult{message: "measurement manual value updated"}, nil failureResult.message = "measurement manual value updated"
return failureResult, nil
default: default:
return measurementUpdateResult{}, fmt.Errorf("unsupported measurement update field %q", field) return measurementUpdateResult{}, fmt.Errorf("unsupported measurement update field %q", field)
} }
@ -383,21 +423,6 @@ func isInvalidDataObjectUpdateError(err error) bool {
errors.Is(err, common.ErrAmbiguousMeasurementToken) errors.Is(err, common.ErrAmbiguousMeasurementToken)
} }
func writeMeasurementManualValue(ctx context.Context, measurement *orm.Measurement, value float64) error {
key, err := model.GenerateMeasureIdentifier(measurement.DataSource)
if err != nil {
return fmt.Errorf("generate measurement redis key: %w", err)
}
zset, err := diagram.NewRedisZSet(ctx, key, 0, false)
if err != nil {
return fmt.Errorf("create measurement redis zset: %w", err)
}
if err := zset.ZADD(key, value, strconv.FormatInt(time.Now().UnixNano(), 10)); err != nil {
return fmt.Errorf("write manual measurement value to redis: %w", err)
}
return nil
}
func callRealTimeDataWriteStopInterface(_ context.Context, _ orm.JSONMap, _ *float64) error { func callRealTimeDataWriteStopInterface(_ context.Context, _ orm.JSONMap, _ *float64) error {
// TODO: call the dataRT HTTP API. A nil value stops automatic writes; // TODO: call the dataRT HTTP API. A nil value stops automatic writes;
// a non-nil value writes the supplied manual measurement value. // a non-nil value writes the supplied manual measurement value.
@ -408,18 +433,3 @@ func callRealTimeDataWriteStartInterface(_ context.Context, _ orm.JSONMap, _ *fl
// TODO: call the dataRT HTTP API to start automatic measurement writes. // TODO: call the dataRT HTTP API to start automatic measurement writes.
return nil return nil
} }
func replaceMeasurementRedisValue(ctx context.Context, measurement *orm.Measurement, value float64) error {
key, err := model.GenerateMeasureIdentifier(measurement.DataSource)
if err != nil {
return fmt.Errorf("generate measurement redis key: %w", err)
}
zset, err := diagram.NewRedisZSet(ctx, key, 0, false)
if err != nil {
return fmt.Errorf("create measurement redis zset: %w", err)
}
if err := zset.ZREPLACE(key, value, strconv.FormatInt(time.Now().UnixNano(), 10)); err != nil {
return fmt.Errorf("replace manual measurement value in redis: %w", err)
}
return nil
}

View File

@ -26,6 +26,7 @@ func TestValidateDataObjectAttributeUpdateParameterGroups(t *testing.T) {
"craft", "craft",
"integrity", "integrity",
"behavior", "behavior",
"base_extend",
} }
for _, group := range groups { for _, group := range groups {
@ -46,7 +47,7 @@ func TestValidateDataObjectAttributeUpdateParameterGroups(t *testing.T) {
} }
func TestValidateDataObjectAttributeUpdateRejectsUnsupportedParameterGroups(t *testing.T) { func TestValidateDataObjectAttributeUpdateRejectsUnsupportedParameterGroups(t *testing.T) {
for _, group := range []string{"component", "base_extend"} { for _, group := range []string{"component"} {
t.Run(group, func(t *testing.T) { t.Run(group, func(t *testing.T) {
_, _, _, err := validateDataObjectAttributeUpdate(dataObjectAttributeUpdateRequest{ _, _, _, err := validateDataObjectAttributeUpdate(dataObjectAttributeUpdateRequest{
Token: fmt.Sprintf("nspath.component.%s.attribute", group), Token: fmt.Sprintf("nspath.component.%s.attribute", group),
@ -416,7 +417,9 @@ func TestUpdateMeasurementDataObjectWritesValueInManualMode(t *testing.T) {
assert.True(t, called) assert.True(t, called)
assert.True(t, dataRTCalled) assert.True(t, dataRTCalled)
assert.Contains(t, result.message, "updated") assert.Contains(t, result.message, "updated")
assert.False(t, result.recordFailure) assert.True(t, result.recordFailureOnError)
assert.Equal(t, int64(10), result.measurementID)
assert.Equal(t, float64(15.2), result.value)
require.NoError(t, tx.Rollback().Error) require.NoError(t, tx.Rollback().Error)
require.NoError(t, mock.ExpectationsWereMet()) require.NoError(t, mock.ExpectationsWereMet())
} }
@ -439,7 +442,7 @@ func TestUpdateMeasurementDataObjectReturnsFailureResultAndAppError(t *testing.T
require.Error(t, err) require.Error(t, err)
assert.ErrorIs(t, err, errcode.ErrMeasurementValueUpdateFailed) assert.ErrorIs(t, err, errcode.ErrMeasurementValueUpdateFailed)
assert.ErrorIs(t, err, writeErr) assert.ErrorIs(t, err, writeErr)
assert.True(t, result.recordFailure) assert.True(t, result.recordFailureOnError)
assert.Equal(t, int64(10), result.measurementID) assert.Equal(t, int64(10), result.measurementID)
assert.Equal(t, float64(15.2), result.value) assert.Equal(t, float64(15.2), result.value)
require.NoError(t, tx.Rollback().Error) require.NoError(t, tx.Rollback().Error)

View File

@ -0,0 +1,429 @@
package handler
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"time"
"modelRT/constants"
"modelRT/model"
"modelRT/orm"
"github.com/redis/go-redis/v9"
)
const redisChangeRestoreTimeout = 5 * time.Second
type redisHashChange struct {
key string
field string
oldValue string
newValue string
}
type redisZSetChange struct {
key string
oldValues []redis.Z
newValues []redis.Z
}
// RedisChangeSet keeps the Redis changes belonging to one PostgreSQL
// transaction. Changes are prepared first and applied together immediately
// before the PostgreSQL transaction is committed.
type RedisChangeSet struct {
client *redis.Client
hashChanges []redisHashChange
zsetChanges []redisZSetChange
applied bool
}
func NewRedisChangeSet(client *redis.Client) *RedisChangeSet {
return &RedisChangeSet{client: client}
}
func (changes *RedisChangeSet) AddDataObjectHashChange(
ctx context.Context,
dataObjectType constants.DataObjectType,
token, field string,
value any,
) error {
if changes == nil || changes.client == nil {
return fmt.Errorf("redis client is not initialized")
}
metadata, err := changes.client.HMGet(ctx, token, "id", "name").Result()
if err != nil {
return fmt.Errorf("query redis data-object aliases for %q: %w", token, err)
}
if len(metadata) != 2 || metadata[0] == nil || metadata[1] == nil {
return fmt.Errorf("redis data-object hash %q does not contain id and name", token)
}
id, ok := metadata[0].(string)
if !ok {
return fmt.Errorf("redis data-object hash %q id has type %T", token, metadata[0])
}
name, ok := metadata[1].(string)
if !ok {
return fmt.Errorf("redis data-object hash %q name has type %T", token, metadata[1])
}
keys, err := dataObjectRedisAliasKeys(dataObjectType, id, name)
if err != nil {
return err
}
if !containsString(keys, token) {
return fmt.Errorf("redis data-object hash %q metadata points to different aliases", token)
}
newValue, err := redisChangeString(value)
if err != nil {
return fmt.Errorf("encode redis data-object value: %w", err)
}
for _, key := range keys {
keyType, err := changes.client.Type(ctx, key).Result()
if err != nil {
return fmt.Errorf("query redis key type for %q: %w", key, err)
}
if keyType == "none" && dataObjectType == constants.DataObjectTypeParameter && key == name && key != token {
// A parameter short alias is only initialized for a local station.
continue
}
if keyType != "hash" {
return fmt.Errorf("redis data-object key %q has type %q, expected hash", key, keyType)
}
oldValue, err := changes.client.HGet(ctx, key, field).Result()
if errors.Is(err, redis.Nil) {
return fmt.Errorf("redis data-object hash %q does not contain field %q", key, field)
}
if err != nil {
return fmt.Errorf("query redis hash %q field %q: %w", key, field, err)
}
changes.hashChanges = append(changes.hashChanges, redisHashChange{
key: key,
field: field,
oldValue: oldValue,
newValue: newValue,
})
}
return nil
}
func (changes *RedisChangeSet) AddMeasurementValueChange(
ctx context.Context,
measurement *orm.Measurement,
value float64,
replace bool,
) error {
if changes == nil || changes.client == nil {
return fmt.Errorf("redis client is not initialized")
}
if measurement == nil {
return fmt.Errorf("measurement is nil")
}
key, err := model.GenerateMeasureIdentifier(measurement.DataSource)
if err != nil {
return fmt.Errorf("generate measurement redis key: %w", err)
}
keyType, err := changes.client.Type(ctx, key).Result()
if err != nil {
return fmt.Errorf("query measurement redis key type for %q: %w", key, err)
}
if keyType != "none" && keyType != "zset" {
return fmt.Errorf("measurement redis key %q has type %q, expected zset", key, keyType)
}
oldValues, err := changes.client.ZRangeWithScores(ctx, key, 0, -1).Result()
if err != nil {
return fmt.Errorf("query measurement redis values for %q: %w", key, err)
}
newMember := strconv.FormatInt(time.Now().UnixNano(), 10)
newValues := []redis.Z{{Score: value, Member: newMember}}
if !replace {
newValues = mergeRedisZValues(oldValues, newValues...)
}
changes.zsetChanges = append(changes.zsetChanges, redisZSetChange{
key: key,
oldValues: normalizeRedisZValues(oldValues),
newValues: normalizeRedisZValues(newValues),
})
return nil
}
func (changes *RedisChangeSet) Apply(ctx context.Context) error {
if changes == nil || changes.client == nil {
return fmt.Errorf("redis client is not initialized")
}
if len(changes.hashChanges) == 0 && len(changes.zsetChanges) == 0 {
return nil
}
keys := changes.keys()
err := changes.client.Watch(ctx, func(tx *redis.Tx) error {
if err := changes.verify(ctx, tx, false); err != nil {
return err
}
_, err := tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
for _, change := range changes.hashChanges {
pipe.HSet(ctx, change.key, change.field, change.newValue)
}
for _, change := range changes.zsetChanges {
pipe.Del(ctx, change.key)
if len(change.newValues) > 0 {
pipe.ZAdd(ctx, change.key, change.newValues...)
}
}
return nil
})
return err
}, keys...)
if err != nil {
// A connection error can leave EXEC's outcome unknown. Restore only
// when Redis still contains either the prepared or the applied state.
restoreCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), redisChangeRestoreTimeout)
defer cancel()
if restoreErr := changes.restoreAfterApplyFailure(restoreCtx); restoreErr != nil {
return fmt.Errorf("apply redis changes: %w; restore redis changes: %v", err, restoreErr)
}
return fmt.Errorf("apply redis changes: %w", err)
}
changes.applied = true
return nil
}
// Revert restores Redis after a PostgreSQL commit failure. It uses WATCH and
// only restores values that still match this change set.
func (changes *RedisChangeSet) Revert(ctx context.Context) error {
if changes == nil || !changes.applied {
return nil
}
return changes.restoreOldValues(ctx, true)
}
func (changes *RedisChangeSet) restoreOldValues(ctx context.Context, compareNew bool) error {
keys := changes.keys()
return changes.client.Watch(ctx, func(tx *redis.Tx) error {
if compareNew {
if err := changes.verify(ctx, tx, true); err != nil {
return err
}
}
_, err := tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
for _, change := range changes.hashChanges {
pipe.HSet(ctx, change.key, change.field, change.oldValue)
}
for _, change := range changes.zsetChanges {
pipe.Del(ctx, change.key)
if len(change.oldValues) > 0 {
pipe.ZAdd(ctx, change.key, change.oldValues...)
}
}
return nil
})
return err
}, keys...)
}
func (changes *RedisChangeSet) restoreAfterApplyFailure(ctx context.Context) error {
keys := changes.keys()
return changes.client.Watch(ctx, func(tx *redis.Tx) error {
for _, change := range changes.hashChanges {
actual, err := tx.HGet(ctx, change.key, change.field).Result()
if err != nil {
return fmt.Errorf("verify redis hash %q field %q after apply failure: %w", change.key, change.field, err)
}
if actual != change.oldValue && actual != change.newValue {
return fmt.Errorf("redis hash %q field %q changed concurrently", change.key, change.field)
}
}
for _, change := range changes.zsetChanges {
actual, err := tx.ZRangeWithScores(ctx, change.key, 0, -1).Result()
if err != nil {
return fmt.Errorf("verify redis zset %q after apply failure: %w", change.key, err)
}
if !equalRedisZValues(actual, change.oldValues) && !equalRedisZValues(actual, change.newValues) {
return fmt.Errorf("redis zset %q changed concurrently", change.key)
}
}
_, err := tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
for _, change := range changes.hashChanges {
pipe.HSet(ctx, change.key, change.field, change.oldValue)
}
for _, change := range changes.zsetChanges {
pipe.Del(ctx, change.key)
if len(change.oldValues) > 0 {
pipe.ZAdd(ctx, change.key, change.oldValues...)
}
}
return nil
})
return err
}, keys...)
}
func (changes *RedisChangeSet) verify(ctx context.Context, tx *redis.Tx, expectNew bool) error {
for _, change := range changes.hashChanges {
expected := change.oldValue
if expectNew {
expected = change.newValue
}
actual, err := tx.HGet(ctx, change.key, change.field).Result()
if err != nil {
return fmt.Errorf("verify redis hash %q field %q: %w", change.key, change.field, err)
}
if actual != expected {
return fmt.Errorf("redis hash %q field %q changed concurrently", change.key, change.field)
}
}
for _, change := range changes.zsetChanges {
expected := change.oldValues
if expectNew {
expected = change.newValues
}
actual, err := tx.ZRangeWithScores(ctx, change.key, 0, -1).Result()
if err != nil {
return fmt.Errorf("verify redis zset %q: %w", change.key, err)
}
if !equalRedisZValues(actual, expected) {
return fmt.Errorf("redis zset %q changed concurrently", change.key)
}
}
return nil
}
func (changes *RedisChangeSet) keys() []string {
seen := make(map[string]struct{}, len(changes.hashChanges)+len(changes.zsetChanges))
keys := make([]string, 0, len(seen))
for _, change := range changes.hashChanges {
if _, ok := seen[change.key]; !ok {
seen[change.key] = struct{}{}
keys = append(keys, change.key)
}
}
for _, change := range changes.zsetChanges {
if _, ok := seen[change.key]; !ok {
seen[change.key] = struct{}{}
keys = append(keys, change.key)
}
}
sort.Strings(keys)
return keys
}
func dataObjectRedisAliasKeys(dataObjectType constants.DataObjectType, id, name string) ([]string, error) {
switch dataObjectType {
case constants.DataObjectTypeParameter:
if len(strings.Split(id, ".")) != 7 || len(strings.Split(name, ".")) != 4 {
return nil, fmt.Errorf("invalid parameter redis aliases id=%q name=%q", id, name)
}
return uniqueStrings(id, name), nil
case constants.DataObjectTypeMeasurement:
parts := strings.Split(id, ".")
if len(parts) != 7 || len(strings.Split(name, ".")) != 2 {
return nil, fmt.Errorf("invalid measurement redis aliases id=%q name=%q", id, name)
}
return uniqueStrings(id, strings.Join(parts[3:], "."), name), nil
default:
return nil, fmt.Errorf("unsupported data object type %q", dataObjectType)
}
}
func redisChangeString(value any) (string, error) {
switch typedValue := value.(type) {
case string:
return typedValue, nil
case []byte:
return string(typedValue), nil
case nil:
return "null", nil
case bool:
return strconv.FormatBool(typedValue), nil
case int:
return strconv.Itoa(typedValue), nil
case int16:
return strconv.FormatInt(int64(typedValue), 10), nil
case int64:
return strconv.FormatInt(typedValue, 10), nil
case float64:
return strconv.FormatFloat(typedValue, 'f', -1, 64), nil
default:
encoded, err := json.Marshal(typedValue)
if err != nil {
return "", err
}
return string(encoded), nil
}
}
func cloneRedisZValues(values []redis.Z) []redis.Z {
cloned := make([]redis.Z, len(values))
copy(cloned, values)
return cloned
}
func mergeRedisZValues(current []redis.Z, additions ...redis.Z) []redis.Z {
valuesByMember := make(map[string]redis.Z, len(current)+len(additions))
for _, value := range current {
valuesByMember[fmt.Sprint(value.Member)] = value
}
for _, value := range additions {
valuesByMember[fmt.Sprint(value.Member)] = value
}
values := make([]redis.Z, 0, len(valuesByMember))
for _, value := range valuesByMember {
values = append(values, value)
}
return normalizeRedisZValues(values)
}
func normalizeRedisZValues(values []redis.Z) []redis.Z {
normalized := cloneRedisZValues(values)
sort.Slice(normalized, func(i, j int) bool {
if normalized[i].Score != normalized[j].Score {
return normalized[i].Score < normalized[j].Score
}
return fmt.Sprint(normalized[i].Member) < fmt.Sprint(normalized[j].Member)
})
return normalized
}
func equalRedisZValues(left, right []redis.Z) bool {
left = normalizeRedisZValues(left)
right = normalizeRedisZValues(right)
if len(left) != len(right) {
return false
}
for index := range left {
if left[index].Score != right[index].Score ||
fmt.Sprint(left[index].Member) != fmt.Sprint(right[index].Member) {
return false
}
}
return true
}
func uniqueStrings(values ...string) []string {
seen := make(map[string]struct{}, len(values))
result := make([]string, 0, len(values))
for _, value := range values {
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
result = append(result, value)
}
return result
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}

View File

@ -0,0 +1,79 @@
package handler
import (
"testing"
"modelRT/constants"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDataObjectRedisAliasKeys(t *testing.T) {
parameterKeys, err := dataObjectRedisAliasKeys(
constants.DataObjectTypeParameter,
"grid.zone.station.nspath.component.rated.voltage",
"nspath.component.rated.voltage",
)
require.NoError(t, err)
assert.Equal(t, []string{
"grid.zone.station.nspath.component.rated.voltage",
"nspath.component.rated.voltage",
}, parameterKeys)
measurementKeys, err := dataObjectRedisAliasKeys(
constants.DataObjectTypeMeasurement,
"grid.zone.station.nspath.component.bay.current",
"nspath.current",
)
require.NoError(t, err)
assert.Equal(t, []string{
"grid.zone.station.nspath.component.bay.current",
"nspath.component.bay.current",
"nspath.current",
}, measurementKeys)
}
func TestDataObjectRedisAliasKeysRejectsInvalidMetadata(t *testing.T) {
_, err := dataObjectRedisAliasKeys(constants.DataObjectTypeParameter, "short.id", "short.name")
require.Error(t, err)
_, err = dataObjectRedisAliasKeys(constants.DataObjectTypeMeasurement, "short.id", "short.name")
require.Error(t, err)
}
func TestRedisChangeStringUsesCacheRepresentations(t *testing.T) {
tests := []struct {
name string
value any
want string
}{
{name: "string", value: "15.2", want: "15.2"},
{name: "integer", value: int64(15), want: "15"},
{name: "decimal", value: 15.2, want: "15.2"},
{name: "boolean", value: true, want: "true"},
{name: "object", value: map[string]any{"enabled": true}, want: `{"enabled":true}`},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := redisChangeString(test.value)
require.NoError(t, err)
assert.Equal(t, test.want, got)
})
}
}
func TestMergeRedisZValuesReplacesDuplicateMemberAndSorts(t *testing.T) {
values := mergeRedisZValues(
[]redis.Z{
{Score: 20, Member: "2"},
{Score: 10, Member: "1"},
},
redis.Z{Score: 30, Member: "1"},
)
assert.Equal(t, []redis.Z{
{Score: 20, Member: "2"},
{Score: 30, Member: "1"},
}, values)
}

View File

@ -73,6 +73,13 @@ func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord)
record.MeasurementMode, record.MeasurementMode,
) )
} }
if record.MeasurementSize <= 0 {
return nil, fmt.Errorf(
"measurement %q window size must be greater than 0, got %d",
record.MeasurementTag,
record.MeasurementSize,
)
}
if record.MeasurementDataSource == nil || if record.MeasurementDataSource == nil ||
record.MeasurementEventPlan == nil || record.MeasurementEventPlan == nil ||
record.MeasurementBinding == nil { record.MeasurementBinding == nil {

View File

@ -65,6 +65,15 @@ func TestBuildMeasurementDataObjectHashesRejectsUnsupportedMeasurementType(t *te
assert.Contains(t, err.Error(), "unsupported measurement type 5") assert.Contains(t, err.Error(), "unsupported measurement type 5")
} }
func TestBuildMeasurementDataObjectHashesRejectsInvalidWindowSize(t *testing.T) {
record := measurementInitializationRecordForTest()
record.MeasurementSize = 0
_, err := buildMeasurementDataObjectHashes([]MeasurementInitializationRecord{record})
require.Error(t, err)
assert.Contains(t, err.Error(), "window size must be greater than 0")
}
func measurementInitializationRecordForTest() MeasurementInitializationRecord { func measurementInitializationRecordForTest() MeasurementInitializationRecord {
return MeasurementInitializationRecord{ return MeasurementInitializationRecord{
GridTag: "grid000", GridTag: "grid000",