refactor(redis): normalize data object storage with aliases

- store one canonical hash for each parameter and measurement
- map supported token formats to canonical Redis keys through aliases
- resolve aliases in query and update handlers
- clean stale hashes and aliases during initialization
- update Redis change logic and related tests
This commit is contained in:
douxu 2026-07-31 13:39:18 +08:00
parent 0b004471fe
commit d670963854
11 changed files with 308 additions and 255 deletions

View File

@ -9,7 +9,19 @@ const (
// startup so stale parameter data-object keys can be removed safely. // startup so stale parameter data-object keys can be removed safely.
RedisParameterDataObjectKeySet = "modelrt:parameter-data-object:keys" RedisParameterDataObjectKeySet = "modelrt:parameter-data-object:keys"
// RedisParameterDataObjectAliasKeySet tracks parameter alias string keys.
RedisParameterDataObjectAliasKeySet = "modelrt:parameter-data-object:alias-keys"
// RedisParameterDataObjectAliasPrefix prefixes parameter token aliases.
RedisParameterDataObjectAliasPrefix = "modelrt:data-object:alias:parameter:"
// RedisMeasurementDataObjectKeySet tracks measurement hashes created during // RedisMeasurementDataObjectKeySet tracks measurement hashes created during
// startup so stale measurement data-object keys can be removed safely. // startup so stale measurement data-object keys can be removed safely.
RedisMeasurementDataObjectKeySet = "modelrt:measurement-data-object:keys" RedisMeasurementDataObjectKeySet = "modelrt:measurement-data-object:keys"
// RedisMeasurementDataObjectAliasKeySet tracks measurement alias string keys.
RedisMeasurementDataObjectAliasKeySet = "modelrt:measurement-data-object:alias-keys"
// RedisMeasurementDataObjectAliasPrefix prefixes measurement token aliases.
RedisMeasurementDataObjectAliasPrefix = "modelrt:data-object:alias:measurement:"
) )

View File

@ -196,26 +196,16 @@ func loadDataObjectHashField(
if rdb == nil { if rdb == nil {
return "", fmt.Errorf("redis client is not initialized") return "", fmt.Errorf("redis client is not initialized")
} }
value, err := rdb.HGet(ctx, token, field).Result() canonicalKey, err := model.ResolveDataObjectRedisKey(ctx, rdb, dataObjectType, token)
if err != nil {
return "", err
}
value, err := rdb.HGet(ctx, canonicalKey, field).Result()
if errors.Is(err, redis.Nil) { if errors.Is(err, redis.Nil) {
exists, existsErr := rdb.Exists(ctx, token).Result() return "", fmt.Errorf("canonical redis data-object hash %q does not contain field %q", canonicalKey, field)
if existsErr != nil {
return "", fmt.Errorf("check redis data-object hash %q: %w", token, existsErr)
}
if exists > 0 {
return "", fmt.Errorf("redis data-object hash %q does not contain field %q", token, field)
}
switch dataObjectType {
case constants.DataObjectTypeParameter:
return "", fmt.Errorf("%w: %q", common.ErrParameterTokenNotFound, token)
case constants.DataObjectTypeMeasurement:
return "", fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
default:
return "", fmt.Errorf("invalid data object type %q", dataObjectType)
}
} }
if err != nil { if err != nil {
return "", fmt.Errorf("query redis hash %q field %q: %w", token, field, err) return "", fmt.Errorf("query canonical redis hash %q field %q: %w", canonicalKey, field, err)
} }
return value, nil return value, nil
} }
@ -225,23 +215,18 @@ func loadMeasurementValueMetadata(ctx context.Context, token string) (orm.JSONMa
if rdb == nil { if rdb == nil {
return nil, 0, fmt.Errorf("redis client is not initialized") return nil, 0, fmt.Errorf("redis client is not initialized")
} }
canonicalKey, err := model.ResolveDataObjectRedisKey(ctx, rdb, constants.DataObjectTypeMeasurement, token)
values, err := rdb.HMGet(ctx, token, "data_source", "size").Result()
if err != nil { if err != nil {
return nil, 0, fmt.Errorf("query redis hash %q measurement value metadata: %w", token, err) return nil, 0, err
}
values, err := rdb.HMGet(ctx, canonicalKey, "data_source", "size").Result()
if err != nil {
return nil, 0, fmt.Errorf("query canonical redis hash %q measurement value metadata: %w", canonicalKey, err)
} }
if len(values) != 2 { if len(values) != 2 {
return nil, 0, fmt.Errorf("redis hash %q returned %d measurement metadata fields", token, len(values)) return nil, 0, fmt.Errorf("canonical redis hash %q returned %d measurement metadata fields", canonicalKey, len(values))
} }
if values[0] == nil || values[1] == nil { if values[0] == nil || values[1] == nil {
exists, existsErr := rdb.Exists(ctx, token).Result()
if existsErr != nil {
return nil, 0, fmt.Errorf("check redis data-object hash %q: %w", token, existsErr)
}
if exists == 0 {
return nil, 0, fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
}
missingFields := make([]string, 0, 2) missingFields := make([]string, 0, 2)
if values[0] == nil { if values[0] == nil {
missingFields = append(missingFields, "data_source") missingFields = append(missingFields, "data_source")
@ -250,24 +235,24 @@ func loadMeasurementValueMetadata(ctx context.Context, token string) (orm.JSONMa
missingFields = append(missingFields, "size") missingFields = append(missingFields, "size")
} }
return nil, 0, fmt.Errorf( return nil, 0, fmt.Errorf(
"redis measurement hash %q does not contain field(s) %s", "canonical redis measurement hash %q does not contain field(s) %s",
token, canonicalKey,
strings.Join(missingFields, ", "), strings.Join(missingFields, ", "),
) )
} }
rawDataSource, ok := values[0].(string) rawDataSource, ok := values[0].(string)
if !ok { if !ok {
return nil, 0, fmt.Errorf("redis measurement hash %q data_source has type %T", token, values[0]) return nil, 0, fmt.Errorf("canonical redis measurement hash %q data_source has type %T", canonicalKey, values[0])
} }
var dataSource orm.JSONMap var dataSource orm.JSONMap
if err := json.Unmarshal([]byte(rawDataSource), &dataSource); err != nil { if err := json.Unmarshal([]byte(rawDataSource), &dataSource); err != nil {
return nil, 0, fmt.Errorf("decode measurement data_source from redis hash %q: %w", token, err) return nil, 0, fmt.Errorf("decode measurement data_source from canonical redis hash %q: %w", canonicalKey, err)
} }
rawSize, ok := values[1].(string) rawSize, ok := values[1].(string)
if !ok { if !ok {
return nil, 0, fmt.Errorf("redis measurement hash %q size has type %T", token, values[1]) return nil, 0, fmt.Errorf("canonical redis measurement hash %q size has type %T", canonicalKey, values[1])
} }
size, err := strconv.Atoi(rawSize) size, err := strconv.Atoi(rawSize)
if err != nil { if err != nil {

View File

@ -61,20 +61,29 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
} }
}() }()
redisChanges := NewRedisChangeSet(diagram.GetRedisClientInstance()) redisClient := diagram.GetRedisClientInstance()
redisChanges := NewRedisChangeSet(redisClient)
canonicalRedisKey, err := model.ResolveDataObjectRedisKey(
ctx,
redisClient,
dataObjectType,
request.Token,
)
message := "data-object attribute update success" message := "data-object attribute update success"
var measurementResult measurementUpdateResult var measurementResult measurementUpdateResult
switch dataObjectType { switch {
case constants.DataObjectTypeParameter: case err != nil:
// The shared resolver error is handled by the common failure path below.
case dataObjectType == constants.DataObjectTypeParameter:
parameter, queryErr := database.QueryParameterByDataObjectToken(ctx, tx, request.Token) parameter, queryErr := database.QueryParameterByDataObjectToken(ctx, tx, request.Token)
if queryErr == nil { if queryErr == nil {
queryErr = database.UpdateParameterDataObjectValue(ctx, tx, parameter, value) queryErr = database.UpdateParameterDataObjectValue(ctx, tx, parameter, value)
} }
if queryErr == nil { if queryErr == nil {
queryErr = redisChanges.AddDataObjectHashChange(ctx, dataObjectType, request.Token, field, value) queryErr = redisChanges.AddHashChange(ctx, canonicalRedisKey, field, value)
} }
err = queryErr err = queryErr
case constants.DataObjectTypeMeasurement: case dataObjectType == constants.DataObjectTypeMeasurement:
measurementResult, err = updateMeasurementDataObject(ctx, tx, request.Token, field, value, request.Data, measurementUpdateDependencies{ measurementResult, err = updateMeasurementDataObject(ctx, tx, request.Token, field, value, request.Data, measurementUpdateDependencies{
writeManualValueFunc: func(ctx context.Context, measurement *orm.Measurement, value float64) error { writeManualValueFunc: func(ctx context.Context, measurement *orm.Measurement, value float64) error {
return redisChanges.AddMeasurementValueChange(ctx, measurement, value, false) return redisChanges.AddMeasurementValueChange(ctx, measurement, value, false)
@ -86,7 +95,7 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
}, },
}) })
if err == nil && measurementResult.modeChanged { if err == nil && measurementResult.modeChanged {
err = redisChanges.AddDataObjectHashChange(ctx, dataObjectType, request.Token, "mode", measurementResult.mode) err = redisChanges.AddHashChange(ctx, canonicalRedisKey, "mode", measurementResult.mode)
} }
message = measurementResult.message message = measurementResult.message
default: default:

View File

@ -3,14 +3,11 @@ package handler
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"sort" "sort"
"strconv" "strconv"
"strings"
"time" "time"
"modelRT/constants"
"modelRT/model" "modelRT/model"
"modelRT/orm" "modelRT/orm"
@ -46,70 +43,32 @@ func NewRedisChangeSet(client *redis.Client) *RedisChangeSet {
return &RedisChangeSet{client: client} return &RedisChangeSet{client: client}
} }
func (changes *RedisChangeSet) AddDataObjectHashChange( func (changes *RedisChangeSet) AddHashChange(
ctx context.Context, ctx context.Context,
dataObjectType constants.DataObjectType, canonicalKey, field string,
token, field string,
value any, value any,
) error { ) error {
if changes == nil || changes.client == nil { if changes == nil || changes.client == nil {
return fmt.Errorf("redis client is not initialized") return fmt.Errorf("redis client is not initialized")
} }
if canonicalKey == "" {
metadata, err := changes.client.HMGet(ctx, token, "id", "name").Result() return fmt.Errorf("canonical redis key is empty")
if err != nil {
return fmt.Errorf("query redis data-object aliases for %q: %w", token, err)
}
if len(metadata) != 2 || metadata[0] == nil || metadata[1] == nil {
return fmt.Errorf("redis data-object hash %q does not contain id and name", token)
}
id, ok := metadata[0].(string)
if !ok {
return fmt.Errorf("redis data-object hash %q id has type %T", token, metadata[0])
}
name, ok := metadata[1].(string)
if !ok {
return fmt.Errorf("redis data-object hash %q name has type %T", token, metadata[1])
}
keys, err := dataObjectRedisAliasKeys(dataObjectType, id, name)
if err != nil {
return err
}
if !containsString(keys, token) {
return fmt.Errorf("redis data-object hash %q metadata points to different aliases", token)
} }
newValue, err := redisChangeString(value) newValue, err := redisChangeString(value)
if err != nil { if err != nil {
return fmt.Errorf("encode redis data-object value: %w", err) return fmt.Errorf("encode redis data-object value: %w", err)
} }
for _, key := range keys { oldValue, err := changes.client.HGet(ctx, canonicalKey, field).Result()
keyType, err := changes.client.Type(ctx, key).Result() if err != nil {
if err != nil { return fmt.Errorf("query canonical redis hash %q field %q: %w", canonicalKey, field, err)
return fmt.Errorf("query redis key type for %q: %w", key, err)
}
if keyType == "none" && dataObjectType == constants.DataObjectTypeParameter && key == name && key != token {
// A parameter short alias is only initialized for a local station.
continue
}
if keyType != "hash" {
return fmt.Errorf("redis data-object key %q has type %q, expected hash", key, keyType)
}
oldValue, err := changes.client.HGet(ctx, key, field).Result()
if errors.Is(err, redis.Nil) {
return fmt.Errorf("redis data-object hash %q does not contain field %q", key, field)
}
if err != nil {
return fmt.Errorf("query redis hash %q field %q: %w", key, field, err)
}
changes.hashChanges = append(changes.hashChanges, redisHashChange{
key: key,
field: field,
oldValue: oldValue,
newValue: newValue,
})
} }
changes.hashChanges = append(changes.hashChanges, redisHashChange{
key: canonicalKey,
field: field,
oldValue: oldValue,
newValue: newValue,
})
return nil return nil
} }
@ -314,24 +273,6 @@ func (changes *RedisChangeSet) keys() []string {
return keys return keys
} }
func dataObjectRedisAliasKeys(dataObjectType constants.DataObjectType, id, name string) ([]string, error) {
switch dataObjectType {
case constants.DataObjectTypeParameter:
if len(strings.Split(id, ".")) != 7 || len(strings.Split(name, ".")) != 4 {
return nil, fmt.Errorf("invalid parameter redis aliases id=%q name=%q", id, name)
}
return uniqueStrings(id, name), nil
case constants.DataObjectTypeMeasurement:
parts := strings.Split(id, ".")
if len(parts) != 7 || len(strings.Split(name, ".")) != 2 {
return nil, fmt.Errorf("invalid measurement redis aliases id=%q name=%q", id, name)
}
return uniqueStrings(id, strings.Join(parts[3:], "."), name), nil
default:
return nil, fmt.Errorf("unsupported data object type %q", dataObjectType)
}
}
func redisChangeString(value any) (string, error) { func redisChangeString(value any) (string, error) {
switch typedValue := value.(type) { switch typedValue := value.(type) {
case string: case string:
@ -405,25 +346,3 @@ func equalRedisZValues(left, right []redis.Z) bool {
} }
return true return true
} }
func uniqueStrings(values ...string) []string {
seen := make(map[string]struct{}, len(values))
result := make([]string, 0, len(values))
for _, value := range values {
if _, ok := seen[value]; ok {
continue
}
seen[value] = struct{}{}
result = append(result, value)
}
return result
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}

View File

@ -3,46 +3,11 @@ package handler
import ( import (
"testing" "testing"
"modelRT/constants"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestDataObjectRedisAliasKeys(t *testing.T) {
parameterKeys, err := dataObjectRedisAliasKeys(
constants.DataObjectTypeParameter,
"grid.zone.station.nspath.component.rated.voltage",
"nspath.component.rated.voltage",
)
require.NoError(t, err)
assert.Equal(t, []string{
"grid.zone.station.nspath.component.rated.voltage",
"nspath.component.rated.voltage",
}, parameterKeys)
measurementKeys, err := dataObjectRedisAliasKeys(
constants.DataObjectTypeMeasurement,
"grid.zone.station.nspath.component.bay.current",
"nspath.current",
)
require.NoError(t, err)
assert.Equal(t, []string{
"grid.zone.station.nspath.component.bay.current",
"nspath.component.bay.current",
"nspath.current",
}, measurementKeys)
}
func TestDataObjectRedisAliasKeysRejectsInvalidMetadata(t *testing.T) {
_, err := dataObjectRedisAliasKeys(constants.DataObjectTypeParameter, "short.id", "short.name")
require.Error(t, err)
_, err = dataObjectRedisAliasKeys(constants.DataObjectTypeMeasurement, "short.id", "short.name")
require.Error(t, err)
}
func TestRedisChangeStringUsesCacheRepresentations(t *testing.T) { func TestRedisChangeStringUsesCacheRepresentations(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View File

@ -0,0 +1,78 @@
package model
import (
"context"
"errors"
"fmt"
"modelRT/common"
"modelRT/constants"
"github.com/redis/go-redis/v9"
)
// DataObjectRedisAliasKey returns the Redis string key used to resolve any
// supported token form to the one canonical full-token hash.
func DataObjectRedisAliasKey(dataObjectType constants.DataObjectType, token string) (string, error) {
switch dataObjectType {
case constants.DataObjectTypeParameter:
return constants.RedisParameterDataObjectAliasPrefix + token, nil
case constants.DataObjectTypeMeasurement:
return constants.RedisMeasurementDataObjectAliasPrefix + token, nil
default:
return "", fmt.Errorf("unsupported data object type %q", dataObjectType)
}
}
// ResolveDataObjectRedisKey resolves a full or short token to the canonical
// full-token Redis hash created during data-object initialization.
func ResolveDataObjectRedisKey(
ctx context.Context,
rdb *redis.Client,
dataObjectType constants.DataObjectType,
token string,
) (string, error) {
if rdb == nil {
return "", fmt.Errorf("redis client is not initialized")
}
classifiedType, err := ClassifyDataObjectToken(token)
if err != nil {
return "", err
}
if classifiedType != dataObjectType {
return "", fmt.Errorf("token %q is %q, expected %q", token, classifiedType, dataObjectType)
}
aliasKey, err := DataObjectRedisAliasKey(dataObjectType, token)
if err != nil {
return "", err
}
canonicalKey, err := rdb.Get(ctx, aliasKey).Result()
if errors.Is(err, redis.Nil) {
switch dataObjectType {
case constants.DataObjectTypeParameter:
return "", fmt.Errorf("%w: %q", common.ErrParameterTokenNotFound, token)
case constants.DataObjectTypeMeasurement:
return "", fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
}
}
if err != nil {
return "", fmt.Errorf("resolve redis data-object alias %q: %w", token, err)
}
if canonicalKey == "" {
return "", fmt.Errorf("redis data-object alias %q points to an empty key", token)
}
keyType, err := rdb.Type(ctx, canonicalKey).Result()
if err != nil {
return "", fmt.Errorf("query canonical redis key type for %q: %w", token, err)
}
if keyType != "hash" {
return "", fmt.Errorf(
"redis data-object alias %q points to key %q with type %q, expected hash",
token,
canonicalKey,
keyType,
)
}
return canonicalKey, nil
}

View File

@ -0,0 +1,25 @@
package model
import (
"testing"
"modelRT/constants"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDataObjectRedisAliasKey(t *testing.T) {
parameterKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeParameter, "nspath.component.rated.voltage")
require.NoError(t, err)
assert.Equal(t, constants.RedisParameterDataObjectAliasPrefix+"nspath.component.rated.voltage", parameterKey)
measurementKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeMeasurement, "nspath.current")
require.NoError(t, err)
assert.Equal(t, constants.RedisMeasurementDataObjectAliasPrefix+"nspath.current", measurementKey)
}
func TestDataObjectRedisAliasKeyRejectsUnsupportedType(t *testing.T) {
_, err := DataObjectRedisAliasKey(constants.DataObjectType("unknown"), "token")
require.Error(t, err)
}

View File

@ -17,8 +17,9 @@ import (
const measurementDataObjectPipelineSize = 500 const measurementDataObjectPipelineSize = 500
type measurementDataObjectHash struct { type measurementDataObjectHash struct {
Key string Key string
Fields map[string]any Aliases []string
Fields map[string]any
} }
// MeasurementInitializationRecord contains a measurement and the hierarchy // MeasurementInitializationRecord contains a measurement and the hierarchy
@ -41,8 +42,8 @@ type MeasurementInitializationRecord struct {
MeasurementBinding orm.JSONMap `gorm:"column:measurement_binding"` MeasurementBinding orm.JSONMap `gorm:"column:measurement_binding"`
} }
// InitializeMeasurementDataObjects creates the seven-part, four-part, and // InitializeMeasurementDataObjects creates one seven-part Redis hash and
// token4.token7 Redis hashes for measurements loaded from PostgreSQL. // aliases for all supported measurement token forms.
func InitializeMeasurementDataObjects(ctx context.Context, records []MeasurementInitializationRecord) error { func InitializeMeasurementDataObjects(ctx context.Context, records []MeasurementInitializationRecord) error {
hashes, err := buildMeasurementDataObjectHashes(records) hashes, err := buildMeasurementDataObjectHashes(records)
if err != nil { if err != nil {
@ -60,7 +61,7 @@ func InitializeMeasurementDataObjects(ctx context.Context, records []Measurement
} }
func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord) ([]measurementDataObjectHash, error) { func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord) ([]measurementDataObjectHash, error) {
hashes := make([]measurementDataObjectHash, 0, len(records)*3) hashes := make([]measurementDataObjectHash, 0, len(records))
seenKeys := make(map[string]string, len(records)*3) seenKeys := make(map[string]string, len(records)*3)
for _, record := range records { for _, record := range records {
if record.MeasurementMode != constants.MeasurementModeManual && if record.MeasurementMode != constants.MeasurementModeManual &&
@ -103,7 +104,8 @@ func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord)
}, ".") }, ".")
twoPartToken := record.ComponentNSPath + "." + record.MeasurementTag twoPartToken := record.ComponentNSPath + "." + record.MeasurementTag
for _, token := range []string{fullToken, fourPartToken, twoPartToken} { aliases := []string{fullToken, fourPartToken, twoPartToken}
for _, token := range aliases {
if err := validateInitializedMeasurementToken(token); err != nil { if err := validateInitializedMeasurementToken(token); err != nil {
return nil, err return nil, err
} }
@ -139,11 +141,18 @@ func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord)
"binding": binding, "binding": binding,
} }
owner := fmt.Sprintf("%d/%s", record.MeasurementID, record.ComponentUUID) owner := fmt.Sprintf("%d/%s", record.MeasurementID, record.ComponentUUID)
for _, token := range []string{fullToken, fourPartToken, twoPartToken} { for _, alias := range aliases {
if err := appendMeasurementDataObjectHash(&hashes, seenKeys, token, owner, fields); err != nil { if existingOwner, exists := seenKeys[alias]; exists {
return nil, err return nil, fmt.Errorf(
"ambiguous measurement token %q is produced by %q and %q",
alias,
existingOwner,
owner,
)
} }
seenKeys[alias] = owner
} }
hashes = append(hashes, measurementDataObjectHash{Key: fullToken, Aliases: aliases, Fields: fields})
} }
return hashes, nil return hashes, nil
} }
@ -159,26 +168,6 @@ func validateInitializedMeasurementToken(token string) error {
return nil return nil
} }
func appendMeasurementDataObjectHash(
hashes *[]measurementDataObjectHash,
seenKeys map[string]string,
key string,
owner string,
fields map[string]any,
) error {
if existingOwner, exists := seenKeys[key]; exists {
return fmt.Errorf(
"ambiguous measurement token %q is produced by %q and %q",
key,
existingOwner,
owner,
)
}
seenKeys[key] = owner
*hashes = append(*hashes, measurementDataObjectHash{Key: key, Fields: fields})
return nil
}
func measurementInitializationJSON(value orm.JSONMap) (string, error) { func measurementInitializationJSON(value orm.JSONMap) (string, error) {
encoded, err := json.Marshal(value) encoded, err := json.Marshal(value)
if err != nil { if err != nil {
@ -200,20 +189,38 @@ func storeMeasurementDataObjectHashes(
if err != nil { if err != nil {
return fmt.Errorf("query previously initialized measurement keys: %w", err) return fmt.Errorf("query previously initialized measurement keys: %w", err)
} }
oldAliasKeys, err := rdb.SMembers(ctx, constants.RedisMeasurementDataObjectAliasKeySet).Result()
if err != nil {
return fmt.Errorf("query previously initialized measurement alias keys: %w", err)
}
currentKeys := make(map[string]struct{}, len(hashes)) currentKeys := make(map[string]struct{}, len(hashes))
currentAliasKeys := make(map[string]struct{}, len(hashes)*3)
for start := 0; start < len(hashes); start += measurementDataObjectPipelineSize { for start := 0; start < len(hashes); start += measurementDataObjectPipelineSize {
end := min(start+measurementDataObjectPipelineSize, len(hashes)) end := min(start+measurementDataObjectPipelineSize, len(hashes))
pipeline := rdb.TxPipeline() pipeline := rdb.TxPipeline()
keyMembers := make([]any, 0, end-start) keyMembers := make([]any, 0, end-start)
aliasKeyMembers := make([]any, 0, (end-start)*3)
for _, hash := range hashes[start:end] { for _, hash := range hashes[start:end] {
pipeline.Del(ctx, hash.Key) pipeline.Del(ctx, hash.Key)
pipeline.HSet(ctx, hash.Key, hash.Fields) pipeline.HSet(ctx, hash.Key, hash.Fields)
keyMembers = append(keyMembers, hash.Key) keyMembers = append(keyMembers, hash.Key)
currentKeys[hash.Key] = struct{}{} currentKeys[hash.Key] = struct{}{}
for _, alias := range hash.Aliases {
aliasKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeMeasurement, alias)
if err != nil {
return err
}
pipeline.Set(ctx, aliasKey, hash.Key, 0)
aliasKeyMembers = append(aliasKeyMembers, aliasKey)
currentAliasKeys[aliasKey] = struct{}{}
}
} }
if len(keyMembers) > 0 { if len(keyMembers) > 0 {
pipeline.SAdd(ctx, constants.RedisMeasurementDataObjectKeySet, keyMembers...) pipeline.SAdd(ctx, constants.RedisMeasurementDataObjectKeySet, keyMembers...)
} }
if len(aliasKeyMembers) > 0 {
pipeline.SAdd(ctx, constants.RedisMeasurementDataObjectAliasKeySet, aliasKeyMembers...)
}
if _, err := pipeline.Exec(ctx); err != nil { if _, err := pipeline.Exec(ctx); err != nil {
return fmt.Errorf("write measurement data-object hash batch starting at %d: %w", start, err) return fmt.Errorf("write measurement data-object hash batch starting at %d: %w", start, err)
} }
@ -235,8 +242,24 @@ func storeMeasurementDataObjectHashes(
} }
cleanupPipeline.SRem(ctx, constants.RedisMeasurementDataObjectKeySet, members...) cleanupPipeline.SRem(ctx, constants.RedisMeasurementDataObjectKeySet, members...)
} }
staleAliasKeys := make([]string, 0)
for _, aliasKey := range oldAliasKeys {
if _, exists := currentAliasKeys[aliasKey]; !exists {
staleAliasKeys = append(staleAliasKeys, aliasKey)
}
}
for start := 0; start < len(staleAliasKeys); start += measurementDataObjectPipelineSize {
end := min(start+measurementDataObjectPipelineSize, len(staleAliasKeys))
cleanupPipeline.Del(ctx, staleAliasKeys[start:end]...)
members := make([]any, 0, end-start)
for _, aliasKey := range staleAliasKeys[start:end] {
members = append(members, aliasKey)
}
cleanupPipeline.SRem(ctx, constants.RedisMeasurementDataObjectAliasKeySet, members...)
}
if len(hashes) == 0 { if len(hashes) == 0 {
cleanupPipeline.Del(ctx, constants.RedisMeasurementDataObjectKeySet) cleanupPipeline.Del(ctx, constants.RedisMeasurementDataObjectKeySet)
cleanupPipeline.Del(ctx, constants.RedisMeasurementDataObjectAliasKeySet)
} }
if _, err := cleanupPipeline.Exec(ctx); err != nil { if _, err := cleanupPipeline.Exec(ctx); err != nil {
return fmt.Errorf("remove stale measurement data-object hashes: %w", err) return fmt.Errorf("remove stale measurement data-object hashes: %w", err)

View File

@ -9,19 +9,18 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestBuildMeasurementDataObjectHashesCreatesAllTokenForms(t *testing.T) { func TestBuildMeasurementDataObjectHashesCreatesCanonicalHashAndAllAliases(t *testing.T) {
record := measurementInitializationRecordForTest() record := measurementInitializationRecordForTest()
hashes, err := buildMeasurementDataObjectHashes([]MeasurementInitializationRecord{record}) hashes, err := buildMeasurementDataObjectHashes([]MeasurementInitializationRecord{record})
require.NoError(t, err) require.NoError(t, err)
require.Len(t, hashes, 3) require.Len(t, hashes, 1)
fullToken := "grid000.zone000.station000.220kV_xuefulu1.CTA.bay.IA_rms" fullToken := "grid000.zone000.station000.220kV_xuefulu1.CTA.bay.IA_rms"
fourPartToken := "220kV_xuefulu1.CTA.bay.IA_rms" fourPartToken := "220kV_xuefulu1.CTA.bay.IA_rms"
twoPartToken := "220kV_xuefulu1.IA_rms" twoPartToken := "220kV_xuefulu1.IA_rms"
assert.Equal(t, fullToken, hashes[0].Key) assert.Equal(t, fullToken, hashes[0].Key)
assert.Equal(t, fourPartToken, hashes[1].Key) assert.Equal(t, []string{fullToken, fourPartToken, twoPartToken}, hashes[0].Aliases)
assert.Equal(t, twoPartToken, hashes[2].Key)
fields := hashes[0].Fields fields := hashes[0].Fields
assert.NotContains(t, fields, "value") assert.NotContains(t, fields, "value")
@ -35,8 +34,6 @@ func TestBuildMeasurementDataObjectHashesCreatesAllTokenForms(t *testing.T) {
assert.Equal(t, `{"io_address":{"channel":"TM1","device":"CTA","dtype":1,"option":"rms","station":"001"},"type":1}`, fields["data_source"]) assert.Equal(t, `{"io_address":{"channel":"TM1","device":"CTA","dtype":1,"option":"rms","station":"001"},"type":1}`, fields["data_source"])
assert.Equal(t, `{}`, fields["event_plan"]) assert.Equal(t, `{}`, fields["event_plan"])
assert.Equal(t, `{"ct":{"index":0,"polarity":1,"ratio":1250}}`, fields["binding"]) assert.Equal(t, `{"ct":{"index":0,"polarity":1,"ratio":1250}}`, fields["binding"])
assert.Equal(t, fields, hashes[1].Fields)
assert.Equal(t, fields, hashes[2].Fields)
} }
func TestBuildMeasurementDataObjectHashesRejectsAmbiguousShortToken(t *testing.T) { func TestBuildMeasurementDataObjectHashesRejectsAmbiguousShortToken(t *testing.T) {

View File

@ -18,8 +18,9 @@ import (
const parameterDataObjectPipelineSize = 500 const parameterDataObjectPipelineSize = 500
type parameterDataObjectHash struct { type parameterDataObjectHash struct {
Key string Key string
Fields map[string]any Aliases []string
Fields map[string]any
} }
// ParameterInitializationRecord contains one parameter attribute together with // ParameterInitializationRecord contains one parameter attribute together with
@ -41,8 +42,8 @@ type ParameterInitializationRecord struct {
DynamicRecordCount int64 `gorm:"column:dynamic_record_count"` DynamicRecordCount int64 `gorm:"column:dynamic_record_count"`
} }
// InitializeParameterDataObjects creates full and local-short Redis hashes for // InitializeParameterDataObjects creates one full-token Redis hash and full or
// parameter attributes previously loaded from PostgreSQL. // local-short token aliases for each parameter loaded from PostgreSQL.
func InitializeParameterDataObjects(ctx context.Context, records []ParameterInitializationRecord) error { func InitializeParameterDataObjects(ctx context.Context, records []ParameterInitializationRecord) error {
hashes, err := buildParameterDataObjectHashes(records) hashes, err := buildParameterDataObjectHashes(records)
if err != nil { if err != nil {
@ -60,7 +61,7 @@ func InitializeParameterDataObjects(ctx context.Context, records []ParameterInit
} }
func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]parameterDataObjectHash, error) { func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]parameterDataObjectHash, error) {
hashes := make([]parameterDataObjectHash, 0, len(records)*2) hashes := make([]parameterDataObjectHash, 0, len(records))
seenKeys := make(map[string]string, len(records)*2) seenKeys := make(map[string]string, len(records)*2)
for _, record := range records { for _, record := range records {
@ -91,8 +92,9 @@ func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]
record.AttributeName, record.AttributeName,
}, ".") }, ".")
if err := validateInitializedParameterToken(fullToken); err != nil { aliases := []string{fullToken}
return nil, err if record.StationIsLocal {
aliases = append(aliases, shortToken)
} }
fields := map[string]any{ fields := map[string]any{
"value": value, "value": value,
@ -107,19 +109,21 @@ func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]
record.AttributeGroup, record.AttributeGroup,
record.AttributeName, record.AttributeName,
}, "/") }, "/")
if err := appendParameterDataObjectHash(&hashes, seenKeys, fullToken, owner, fields); err != nil { for _, alias := range aliases {
return nil, err if err := validateInitializedParameterToken(alias); err != nil {
} return nil, err
}
if !record.StationIsLocal { if existingOwner, exists := seenKeys[alias]; exists {
continue return nil, fmt.Errorf(
} "ambiguous parameter token %q is produced by %q and %q",
if err := validateInitializedParameterToken(shortToken); err != nil { alias,
return nil, err existingOwner,
} owner,
if err := appendParameterDataObjectHash(&hashes, seenKeys, shortToken, owner, fields); err != nil { )
return nil, err }
seenKeys[alias] = owner
} }
hashes = append(hashes, parameterDataObjectHash{Key: fullToken, Aliases: aliases, Fields: fields})
} }
return hashes, nil return hashes, nil
} }
@ -135,26 +139,6 @@ func validateInitializedParameterToken(token string) error {
return nil return nil
} }
func appendParameterDataObjectHash(
hashes *[]parameterDataObjectHash,
seenKeys map[string]string,
key string,
owner string,
fields map[string]any,
) error {
if existingOwner, exists := seenKeys[key]; exists {
return fmt.Errorf(
"ambiguous parameter token %q is produced by %q and %q",
key,
existingOwner,
owner,
)
}
seenKeys[key] = owner
*hashes = append(*hashes, parameterDataObjectHash{Key: key, Fields: fields})
return nil
}
func parameterRedisValue(rawJSON string) (any, error) { func parameterRedisValue(rawJSON string) (any, error) {
decoder := json.NewDecoder(bytes.NewBufferString(rawJSON)) decoder := json.NewDecoder(bytes.NewBufferString(rawJSON))
decoder.UseNumber() decoder.UseNumber()
@ -194,30 +178,86 @@ func storeParameterDataObjectHashes(
if err != nil { if err != nil {
return fmt.Errorf("query previously initialized parameter keys: %w", err) return fmt.Errorf("query previously initialized parameter keys: %w", err)
} }
cleanupPipeline := rdb.TxPipeline() oldAliasKeys, err := rdb.SMembers(ctx, constants.RedisParameterDataObjectAliasKeySet).Result()
for start := 0; start < len(oldKeys); start += parameterDataObjectPipelineSize { if err != nil {
end := min(start+parameterDataObjectPipelineSize, len(oldKeys)) return fmt.Errorf("query previously initialized parameter alias keys: %w", err)
cleanupPipeline.Del(ctx, oldKeys[start:end]...)
}
cleanupPipeline.Del(ctx, constants.RedisParameterDataObjectKeySet)
if _, err := cleanupPipeline.Exec(ctx); err != nil {
return fmt.Errorf("remove stale parameter data-object hashes: %w", err)
} }
currentKeys := make(map[string]struct{}, len(hashes))
currentAliasKeys := make(map[string]struct{}, len(hashes)*2)
for start := 0; start < len(hashes); start += parameterDataObjectPipelineSize { for start := 0; start < len(hashes); start += parameterDataObjectPipelineSize {
end := min(start+parameterDataObjectPipelineSize, len(hashes)) end := min(start+parameterDataObjectPipelineSize, len(hashes))
pipeline := rdb.TxPipeline() pipeline := rdb.TxPipeline()
keyMembers := make([]any, 0, end-start) keyMembers := make([]any, 0, end-start)
aliasKeyMembers := make([]any, 0, (end-start)*2)
for _, hash := range hashes[start:end] { for _, hash := range hashes[start:end] {
pipeline.Del(ctx, hash.Key)
pipeline.HSet(ctx, hash.Key, hash.Fields) pipeline.HSet(ctx, hash.Key, hash.Fields)
keyMembers = append(keyMembers, hash.Key) keyMembers = append(keyMembers, hash.Key)
currentKeys[hash.Key] = struct{}{}
for _, alias := range hash.Aliases {
aliasKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeParameter, alias)
if err != nil {
return err
}
pipeline.Set(ctx, aliasKey, hash.Key, 0)
aliasKeyMembers = append(aliasKeyMembers, aliasKey)
currentAliasKeys[aliasKey] = struct{}{}
}
} }
if len(keyMembers) > 0 { if len(keyMembers) > 0 {
pipeline.SAdd(ctx, constants.RedisParameterDataObjectKeySet, keyMembers...) pipeline.SAdd(ctx, constants.RedisParameterDataObjectKeySet, keyMembers...)
} }
if len(aliasKeyMembers) > 0 {
pipeline.SAdd(ctx, constants.RedisParameterDataObjectAliasKeySet, aliasKeyMembers...)
}
if _, err := pipeline.Exec(ctx); err != nil { if _, err := pipeline.Exec(ctx); err != nil {
return fmt.Errorf("write parameter data-object hash batch starting at %d: %w", start, err) return fmt.Errorf("write parameter data-object hash batch starting at %d: %w", start, err)
} }
} }
if err := cleanupStaleParameterDataObjectKeys(
ctx,
rdb,
oldKeys,
oldAliasKeys,
currentKeys,
currentAliasKeys,
); err != nil {
return err
}
return nil
}
func cleanupStaleParameterDataObjectKeys(
ctx context.Context,
rdb *redis.Client,
oldKeys, oldAliasKeys []string,
currentKeys, currentAliasKeys map[string]struct{},
) error {
pipeline := rdb.TxPipeline()
for _, key := range oldKeys {
if _, exists := currentKeys[key]; exists {
continue
}
pipeline.Del(ctx, key)
pipeline.SRem(ctx, constants.RedisParameterDataObjectKeySet, key)
}
for _, aliasKey := range oldAliasKeys {
if _, exists := currentAliasKeys[aliasKey]; exists {
continue
}
pipeline.Del(ctx, aliasKey)
pipeline.SRem(ctx, constants.RedisParameterDataObjectAliasKeySet, aliasKey)
}
if len(currentKeys) == 0 {
pipeline.Del(ctx, constants.RedisParameterDataObjectKeySet)
}
if len(currentAliasKeys) == 0 {
pipeline.Del(ctx, constants.RedisParameterDataObjectAliasKeySet)
}
if _, err := pipeline.Exec(ctx); err != nil {
return fmt.Errorf("remove stale parameter data-object keys: %w", err)
}
return nil return nil
} }

View File

@ -8,7 +8,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestBuildParameterDataObjectHashesCreatesFullAndLocalKeys(t *testing.T) { func TestBuildParameterDataObjectHashesCreatesCanonicalHashAndLocalAliases(t *testing.T) {
records := []ParameterInitializationRecord{ records := []ParameterInitializationRecord{
{ {
GridTag: "grid000", GridTag: "grid000",
@ -30,19 +30,18 @@ func TestBuildParameterDataObjectHashesCreatesFullAndLocalKeys(t *testing.T) {
hashes, err := buildParameterDataObjectHashes(records) hashes, err := buildParameterDataObjectHashes(records)
require.NoError(t, err) require.NoError(t, err)
require.Len(t, hashes, 2) require.Len(t, hashes, 1)
fullToken := "grid000.zone000.station000.220kV_xuefulu1.cable_22.base_extend.vnom_kv" fullToken := "grid000.zone000.station000.220kV_xuefulu1.cable_22.base_extend.vnom_kv"
shortToken := "220kV_xuefulu1.cable_22.base_extend.vnom_kv" shortToken := "220kV_xuefulu1.cable_22.base_extend.vnom_kv"
assert.Equal(t, fullToken, hashes[0].Key) assert.Equal(t, fullToken, hashes[0].Key)
assert.Equal(t, shortToken, hashes[1].Key) assert.Equal(t, []string{fullToken, shortToken}, hashes[0].Aliases)
assert.Equal(t, "7800.00", hashes[0].Fields["value"]) assert.Equal(t, "7800.00", hashes[0].Fields["value"])
assert.Equal(t, "PARAM", hashes[0].Fields["meta"]) assert.Equal(t, "PARAM", hashes[0].Fields["meta"])
assert.Equal(t, "DOUBLE PRECISION", hashes[0].Fields["type"]) assert.Equal(t, "DOUBLE PRECISION", hashes[0].Fields["type"])
assert.Equal(t, shortToken, hashes[0].Fields["name"]) assert.Equal(t, shortToken, hashes[0].Fields["name"])
assert.Equal(t, "额定电压", hashes[0].Fields["description"]) assert.Equal(t, "额定电压", hashes[0].Fields["description"])
assert.Equal(t, fullToken, hashes[0].Fields["id"]) assert.Equal(t, fullToken, hashes[0].Fields["id"])
assert.Equal(t, hashes[0].Fields, hashes[1].Fields)
} }
func TestBuildParameterDataObjectHashesSkipsShortKeyForNonLocalStation(t *testing.T) { func TestBuildParameterDataObjectHashesSkipsShortKeyForNonLocalStation(t *testing.T) {
@ -69,6 +68,7 @@ func TestBuildParameterDataObjectHashesSkipsShortKeyForNonLocalStation(t *testin
require.NoError(t, err) require.NoError(t, err)
require.Len(t, hashes, 1) require.Len(t, hashes, 1)
assert.Equal(t, "grid.zone.station.nspath.component.stable.attribute", hashes[0].Key) assert.Equal(t, "grid.zone.station.nspath.component.stable.attribute", hashes[0].Key)
assert.Equal(t, []string{"grid.zone.station.nspath.component.stable.attribute"}, hashes[0].Aliases)
assert.Equal(t, true, hashes[0].Fields["value"]) assert.Equal(t, true, hashes[0].Fields["value"])
} }