refactor(redis): normalize data object storage with aliases
- store one canonical hash for each parameter and measurement - map supported token formats to canonical Redis keys through aliases - resolve aliases in query and update handlers - clean stale hashes and aliases during initialization - update Redis change logic and related tests
This commit is contained in:
parent
0b004471fe
commit
d670963854
|
|
@ -9,7 +9,19 @@ const (
|
||||||
// startup so stale parameter data-object keys can be removed safely.
|
// startup so stale parameter data-object keys can be removed safely.
|
||||||
RedisParameterDataObjectKeySet = "modelrt:parameter-data-object:keys"
|
RedisParameterDataObjectKeySet = "modelrt:parameter-data-object:keys"
|
||||||
|
|
||||||
|
// RedisParameterDataObjectAliasKeySet tracks parameter alias string keys.
|
||||||
|
RedisParameterDataObjectAliasKeySet = "modelrt:parameter-data-object:alias-keys"
|
||||||
|
|
||||||
|
// RedisParameterDataObjectAliasPrefix prefixes parameter token aliases.
|
||||||
|
RedisParameterDataObjectAliasPrefix = "modelrt:data-object:alias:parameter:"
|
||||||
|
|
||||||
// RedisMeasurementDataObjectKeySet tracks measurement hashes created during
|
// RedisMeasurementDataObjectKeySet tracks measurement hashes created during
|
||||||
// startup so stale measurement data-object keys can be removed safely.
|
// startup so stale measurement data-object keys can be removed safely.
|
||||||
RedisMeasurementDataObjectKeySet = "modelrt:measurement-data-object:keys"
|
RedisMeasurementDataObjectKeySet = "modelrt:measurement-data-object:keys"
|
||||||
|
|
||||||
|
// RedisMeasurementDataObjectAliasKeySet tracks measurement alias string keys.
|
||||||
|
RedisMeasurementDataObjectAliasKeySet = "modelrt:measurement-data-object:alias-keys"
|
||||||
|
|
||||||
|
// RedisMeasurementDataObjectAliasPrefix prefixes measurement token aliases.
|
||||||
|
RedisMeasurementDataObjectAliasPrefix = "modelrt:data-object:alias:measurement:"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -196,26 +196,16 @@ func loadDataObjectHashField(
|
||||||
if rdb == nil {
|
if rdb == nil {
|
||||||
return "", fmt.Errorf("redis client is not initialized")
|
return "", fmt.Errorf("redis client is not initialized")
|
||||||
}
|
}
|
||||||
value, err := rdb.HGet(ctx, token, field).Result()
|
canonicalKey, err := model.ResolveDataObjectRedisKey(ctx, rdb, dataObjectType, token)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
value, err := rdb.HGet(ctx, canonicalKey, field).Result()
|
||||||
if errors.Is(err, redis.Nil) {
|
if errors.Is(err, redis.Nil) {
|
||||||
exists, existsErr := rdb.Exists(ctx, token).Result()
|
return "", fmt.Errorf("canonical redis data-object hash %q does not contain field %q", canonicalKey, field)
|
||||||
if existsErr != nil {
|
|
||||||
return "", fmt.Errorf("check redis data-object hash %q: %w", token, existsErr)
|
|
||||||
}
|
|
||||||
if exists > 0 {
|
|
||||||
return "", fmt.Errorf("redis data-object hash %q does not contain field %q", token, field)
|
|
||||||
}
|
|
||||||
switch dataObjectType {
|
|
||||||
case constants.DataObjectTypeParameter:
|
|
||||||
return "", fmt.Errorf("%w: %q", common.ErrParameterTokenNotFound, token)
|
|
||||||
case constants.DataObjectTypeMeasurement:
|
|
||||||
return "", fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
|
|
||||||
default:
|
|
||||||
return "", fmt.Errorf("invalid data object type %q", dataObjectType)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("query redis hash %q field %q: %w", token, field, err)
|
return "", fmt.Errorf("query canonical redis hash %q field %q: %w", canonicalKey, field, err)
|
||||||
}
|
}
|
||||||
return value, nil
|
return value, nil
|
||||||
}
|
}
|
||||||
|
|
@ -225,23 +215,18 @@ func loadMeasurementValueMetadata(ctx context.Context, token string) (orm.JSONMa
|
||||||
if rdb == nil {
|
if rdb == nil {
|
||||||
return nil, 0, fmt.Errorf("redis client is not initialized")
|
return nil, 0, fmt.Errorf("redis client is not initialized")
|
||||||
}
|
}
|
||||||
|
canonicalKey, err := model.ResolveDataObjectRedisKey(ctx, rdb, constants.DataObjectTypeMeasurement, token)
|
||||||
values, err := rdb.HMGet(ctx, token, "data_source", "size").Result()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, fmt.Errorf("query redis hash %q measurement value metadata: %w", token, err)
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
values, err := rdb.HMGet(ctx, canonicalKey, "data_source", "size").Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("query canonical redis hash %q measurement value metadata: %w", canonicalKey, err)
|
||||||
}
|
}
|
||||||
if len(values) != 2 {
|
if len(values) != 2 {
|
||||||
return nil, 0, fmt.Errorf("redis hash %q returned %d measurement metadata fields", token, len(values))
|
return nil, 0, fmt.Errorf("canonical redis hash %q returned %d measurement metadata fields", canonicalKey, len(values))
|
||||||
}
|
}
|
||||||
if values[0] == nil || values[1] == nil {
|
if values[0] == nil || values[1] == nil {
|
||||||
exists, existsErr := rdb.Exists(ctx, token).Result()
|
|
||||||
if existsErr != nil {
|
|
||||||
return nil, 0, fmt.Errorf("check redis data-object hash %q: %w", token, existsErr)
|
|
||||||
}
|
|
||||||
if exists == 0 {
|
|
||||||
return nil, 0, fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
|
|
||||||
}
|
|
||||||
|
|
||||||
missingFields := make([]string, 0, 2)
|
missingFields := make([]string, 0, 2)
|
||||||
if values[0] == nil {
|
if values[0] == nil {
|
||||||
missingFields = append(missingFields, "data_source")
|
missingFields = append(missingFields, "data_source")
|
||||||
|
|
@ -250,24 +235,24 @@ func loadMeasurementValueMetadata(ctx context.Context, token string) (orm.JSONMa
|
||||||
missingFields = append(missingFields, "size")
|
missingFields = append(missingFields, "size")
|
||||||
}
|
}
|
||||||
return nil, 0, fmt.Errorf(
|
return nil, 0, fmt.Errorf(
|
||||||
"redis measurement hash %q does not contain field(s) %s",
|
"canonical redis measurement hash %q does not contain field(s) %s",
|
||||||
token,
|
canonicalKey,
|
||||||
strings.Join(missingFields, ", "),
|
strings.Join(missingFields, ", "),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
rawDataSource, ok := values[0].(string)
|
rawDataSource, ok := values[0].(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, 0, fmt.Errorf("redis measurement hash %q data_source has type %T", token, values[0])
|
return nil, 0, fmt.Errorf("canonical redis measurement hash %q data_source has type %T", canonicalKey, values[0])
|
||||||
}
|
}
|
||||||
var dataSource orm.JSONMap
|
var dataSource orm.JSONMap
|
||||||
if err := json.Unmarshal([]byte(rawDataSource), &dataSource); err != nil {
|
if err := json.Unmarshal([]byte(rawDataSource), &dataSource); err != nil {
|
||||||
return nil, 0, fmt.Errorf("decode measurement data_source from redis hash %q: %w", token, err)
|
return nil, 0, fmt.Errorf("decode measurement data_source from canonical redis hash %q: %w", canonicalKey, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rawSize, ok := values[1].(string)
|
rawSize, ok := values[1].(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, 0, fmt.Errorf("redis measurement hash %q size has type %T", token, values[1])
|
return nil, 0, fmt.Errorf("canonical redis measurement hash %q size has type %T", canonicalKey, values[1])
|
||||||
}
|
}
|
||||||
size, err := strconv.Atoi(rawSize)
|
size, err := strconv.Atoi(rawSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -61,20 +61,29 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
redisChanges := NewRedisChangeSet(diagram.GetRedisClientInstance())
|
redisClient := diagram.GetRedisClientInstance()
|
||||||
|
redisChanges := NewRedisChangeSet(redisClient)
|
||||||
|
canonicalRedisKey, err := model.ResolveDataObjectRedisKey(
|
||||||
|
ctx,
|
||||||
|
redisClient,
|
||||||
|
dataObjectType,
|
||||||
|
request.Token,
|
||||||
|
)
|
||||||
message := "data-object attribute update success"
|
message := "data-object attribute update success"
|
||||||
var measurementResult measurementUpdateResult
|
var measurementResult measurementUpdateResult
|
||||||
switch dataObjectType {
|
switch {
|
||||||
case constants.DataObjectTypeParameter:
|
case err != nil:
|
||||||
|
// The shared resolver error is handled by the common failure path below.
|
||||||
|
case dataObjectType == constants.DataObjectTypeParameter:
|
||||||
parameter, queryErr := database.QueryParameterByDataObjectToken(ctx, tx, request.Token)
|
parameter, queryErr := database.QueryParameterByDataObjectToken(ctx, tx, request.Token)
|
||||||
if queryErr == nil {
|
if queryErr == nil {
|
||||||
queryErr = database.UpdateParameterDataObjectValue(ctx, tx, parameter, value)
|
queryErr = database.UpdateParameterDataObjectValue(ctx, tx, parameter, value)
|
||||||
}
|
}
|
||||||
if queryErr == nil {
|
if queryErr == nil {
|
||||||
queryErr = redisChanges.AddDataObjectHashChange(ctx, dataObjectType, request.Token, field, value)
|
queryErr = redisChanges.AddHashChange(ctx, canonicalRedisKey, field, value)
|
||||||
}
|
}
|
||||||
err = queryErr
|
err = queryErr
|
||||||
case constants.DataObjectTypeMeasurement:
|
case dataObjectType == constants.DataObjectTypeMeasurement:
|
||||||
measurementResult, err = updateMeasurementDataObject(ctx, tx, request.Token, field, value, request.Data, measurementUpdateDependencies{
|
measurementResult, err = updateMeasurementDataObject(ctx, tx, request.Token, field, value, request.Data, measurementUpdateDependencies{
|
||||||
writeManualValueFunc: func(ctx context.Context, measurement *orm.Measurement, value float64) error {
|
writeManualValueFunc: func(ctx context.Context, measurement *orm.Measurement, value float64) error {
|
||||||
return redisChanges.AddMeasurementValueChange(ctx, measurement, value, false)
|
return redisChanges.AddMeasurementValueChange(ctx, measurement, value, false)
|
||||||
|
|
@ -86,7 +95,7 @@ func DataObjectAttributeUpdateHandler(c *gin.Context) {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err == nil && measurementResult.modeChanged {
|
if err == nil && measurementResult.modeChanged {
|
||||||
err = redisChanges.AddDataObjectHashChange(ctx, dataObjectType, request.Token, "mode", measurementResult.mode)
|
err = redisChanges.AddHashChange(ctx, canonicalRedisKey, "mode", measurementResult.mode)
|
||||||
}
|
}
|
||||||
message = measurementResult.message
|
message = measurementResult.message
|
||||||
default:
|
default:
|
||||||
|
|
|
||||||
|
|
@ -3,14 +3,11 @@ package handler
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"modelRT/constants"
|
|
||||||
"modelRT/model"
|
"modelRT/model"
|
||||||
"modelRT/orm"
|
"modelRT/orm"
|
||||||
|
|
||||||
|
|
@ -46,70 +43,32 @@ func NewRedisChangeSet(client *redis.Client) *RedisChangeSet {
|
||||||
return &RedisChangeSet{client: client}
|
return &RedisChangeSet{client: client}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (changes *RedisChangeSet) AddDataObjectHashChange(
|
func (changes *RedisChangeSet) AddHashChange(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
dataObjectType constants.DataObjectType,
|
canonicalKey, field string,
|
||||||
token, field string,
|
|
||||||
value any,
|
value any,
|
||||||
) error {
|
) error {
|
||||||
if changes == nil || changes.client == nil {
|
if changes == nil || changes.client == nil {
|
||||||
return fmt.Errorf("redis client is not initialized")
|
return fmt.Errorf("redis client is not initialized")
|
||||||
}
|
}
|
||||||
|
if canonicalKey == "" {
|
||||||
metadata, err := changes.client.HMGet(ctx, token, "id", "name").Result()
|
return fmt.Errorf("canonical redis key is empty")
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("query redis data-object aliases for %q: %w", token, err)
|
|
||||||
}
|
|
||||||
if len(metadata) != 2 || metadata[0] == nil || metadata[1] == nil {
|
|
||||||
return fmt.Errorf("redis data-object hash %q does not contain id and name", token)
|
|
||||||
}
|
|
||||||
|
|
||||||
id, ok := metadata[0].(string)
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("redis data-object hash %q id has type %T", token, metadata[0])
|
|
||||||
}
|
|
||||||
name, ok := metadata[1].(string)
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("redis data-object hash %q name has type %T", token, metadata[1])
|
|
||||||
}
|
|
||||||
keys, err := dataObjectRedisAliasKeys(dataObjectType, id, name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !containsString(keys, token) {
|
|
||||||
return fmt.Errorf("redis data-object hash %q metadata points to different aliases", token)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
newValue, err := redisChangeString(value)
|
newValue, err := redisChangeString(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("encode redis data-object value: %w", err)
|
return fmt.Errorf("encode redis data-object value: %w", err)
|
||||||
}
|
}
|
||||||
for _, key := range keys {
|
oldValue, err := changes.client.HGet(ctx, canonicalKey, field).Result()
|
||||||
keyType, err := changes.client.Type(ctx, key).Result()
|
if err != nil {
|
||||||
if err != nil {
|
return fmt.Errorf("query canonical redis hash %q field %q: %w", canonicalKey, field, err)
|
||||||
return fmt.Errorf("query redis key type for %q: %w", key, err)
|
|
||||||
}
|
|
||||||
if keyType == "none" && dataObjectType == constants.DataObjectTypeParameter && key == name && key != token {
|
|
||||||
// A parameter short alias is only initialized for a local station.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if keyType != "hash" {
|
|
||||||
return fmt.Errorf("redis data-object key %q has type %q, expected hash", key, keyType)
|
|
||||||
}
|
|
||||||
oldValue, err := changes.client.HGet(ctx, key, field).Result()
|
|
||||||
if errors.Is(err, redis.Nil) {
|
|
||||||
return fmt.Errorf("redis data-object hash %q does not contain field %q", key, field)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("query redis hash %q field %q: %w", key, field, err)
|
|
||||||
}
|
|
||||||
changes.hashChanges = append(changes.hashChanges, redisHashChange{
|
|
||||||
key: key,
|
|
||||||
field: field,
|
|
||||||
oldValue: oldValue,
|
|
||||||
newValue: newValue,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
changes.hashChanges = append(changes.hashChanges, redisHashChange{
|
||||||
|
key: canonicalKey,
|
||||||
|
field: field,
|
||||||
|
oldValue: oldValue,
|
||||||
|
newValue: newValue,
|
||||||
|
})
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -314,24 +273,6 @@ func (changes *RedisChangeSet) keys() []string {
|
||||||
return keys
|
return keys
|
||||||
}
|
}
|
||||||
|
|
||||||
func dataObjectRedisAliasKeys(dataObjectType constants.DataObjectType, id, name string) ([]string, error) {
|
|
||||||
switch dataObjectType {
|
|
||||||
case constants.DataObjectTypeParameter:
|
|
||||||
if len(strings.Split(id, ".")) != 7 || len(strings.Split(name, ".")) != 4 {
|
|
||||||
return nil, fmt.Errorf("invalid parameter redis aliases id=%q name=%q", id, name)
|
|
||||||
}
|
|
||||||
return uniqueStrings(id, name), nil
|
|
||||||
case constants.DataObjectTypeMeasurement:
|
|
||||||
parts := strings.Split(id, ".")
|
|
||||||
if len(parts) != 7 || len(strings.Split(name, ".")) != 2 {
|
|
||||||
return nil, fmt.Errorf("invalid measurement redis aliases id=%q name=%q", id, name)
|
|
||||||
}
|
|
||||||
return uniqueStrings(id, strings.Join(parts[3:], "."), name), nil
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported data object type %q", dataObjectType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func redisChangeString(value any) (string, error) {
|
func redisChangeString(value any) (string, error) {
|
||||||
switch typedValue := value.(type) {
|
switch typedValue := value.(type) {
|
||||||
case string:
|
case string:
|
||||||
|
|
@ -405,25 +346,3 @@ func equalRedisZValues(left, right []redis.Z) bool {
|
||||||
}
|
}
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func uniqueStrings(values ...string) []string {
|
|
||||||
seen := make(map[string]struct{}, len(values))
|
|
||||||
result := make([]string, 0, len(values))
|
|
||||||
for _, value := range values {
|
|
||||||
if _, ok := seen[value]; ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[value] = struct{}{}
|
|
||||||
result = append(result, value)
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
func containsString(values []string, target string) bool {
|
|
||||||
for _, value := range values {
|
|
||||||
if value == target {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -3,46 +3,11 @@ package handler
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"modelRT/constants"
|
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDataObjectRedisAliasKeys(t *testing.T) {
|
|
||||||
parameterKeys, err := dataObjectRedisAliasKeys(
|
|
||||||
constants.DataObjectTypeParameter,
|
|
||||||
"grid.zone.station.nspath.component.rated.voltage",
|
|
||||||
"nspath.component.rated.voltage",
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []string{
|
|
||||||
"grid.zone.station.nspath.component.rated.voltage",
|
|
||||||
"nspath.component.rated.voltage",
|
|
||||||
}, parameterKeys)
|
|
||||||
|
|
||||||
measurementKeys, err := dataObjectRedisAliasKeys(
|
|
||||||
constants.DataObjectTypeMeasurement,
|
|
||||||
"grid.zone.station.nspath.component.bay.current",
|
|
||||||
"nspath.current",
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []string{
|
|
||||||
"grid.zone.station.nspath.component.bay.current",
|
|
||||||
"nspath.component.bay.current",
|
|
||||||
"nspath.current",
|
|
||||||
}, measurementKeys)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDataObjectRedisAliasKeysRejectsInvalidMetadata(t *testing.T) {
|
|
||||||
_, err := dataObjectRedisAliasKeys(constants.DataObjectTypeParameter, "short.id", "short.name")
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
_, err = dataObjectRedisAliasKeys(constants.DataObjectTypeMeasurement, "short.id", "short.name")
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRedisChangeStringUsesCacheRepresentations(t *testing.T) {
|
func TestRedisChangeStringUsesCacheRepresentations(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,78 @@
|
||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"modelRT/common"
|
||||||
|
"modelRT/constants"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DataObjectRedisAliasKey returns the Redis string key used to resolve any
|
||||||
|
// supported token form to the one canonical full-token hash.
|
||||||
|
func DataObjectRedisAliasKey(dataObjectType constants.DataObjectType, token string) (string, error) {
|
||||||
|
switch dataObjectType {
|
||||||
|
case constants.DataObjectTypeParameter:
|
||||||
|
return constants.RedisParameterDataObjectAliasPrefix + token, nil
|
||||||
|
case constants.DataObjectTypeMeasurement:
|
||||||
|
return constants.RedisMeasurementDataObjectAliasPrefix + token, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unsupported data object type %q", dataObjectType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveDataObjectRedisKey resolves a full or short token to the canonical
|
||||||
|
// full-token Redis hash created during data-object initialization.
|
||||||
|
func ResolveDataObjectRedisKey(
|
||||||
|
ctx context.Context,
|
||||||
|
rdb *redis.Client,
|
||||||
|
dataObjectType constants.DataObjectType,
|
||||||
|
token string,
|
||||||
|
) (string, error) {
|
||||||
|
if rdb == nil {
|
||||||
|
return "", fmt.Errorf("redis client is not initialized")
|
||||||
|
}
|
||||||
|
classifiedType, err := ClassifyDataObjectToken(token)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if classifiedType != dataObjectType {
|
||||||
|
return "", fmt.Errorf("token %q is %q, expected %q", token, classifiedType, dataObjectType)
|
||||||
|
}
|
||||||
|
|
||||||
|
aliasKey, err := DataObjectRedisAliasKey(dataObjectType, token)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
canonicalKey, err := rdb.Get(ctx, aliasKey).Result()
|
||||||
|
if errors.Is(err, redis.Nil) {
|
||||||
|
switch dataObjectType {
|
||||||
|
case constants.DataObjectTypeParameter:
|
||||||
|
return "", fmt.Errorf("%w: %q", common.ErrParameterTokenNotFound, token)
|
||||||
|
case constants.DataObjectTypeMeasurement:
|
||||||
|
return "", fmt.Errorf("%w: %q", common.ErrMeasurementTokenNotFound, token)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("resolve redis data-object alias %q: %w", token, err)
|
||||||
|
}
|
||||||
|
if canonicalKey == "" {
|
||||||
|
return "", fmt.Errorf("redis data-object alias %q points to an empty key", token)
|
||||||
|
}
|
||||||
|
keyType, err := rdb.Type(ctx, canonicalKey).Result()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("query canonical redis key type for %q: %w", token, err)
|
||||||
|
}
|
||||||
|
if keyType != "hash" {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"redis data-object alias %q points to key %q with type %q, expected hash",
|
||||||
|
token,
|
||||||
|
canonicalKey,
|
||||||
|
keyType,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return canonicalKey, nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
package model
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"modelRT/constants"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDataObjectRedisAliasKey(t *testing.T) {
|
||||||
|
parameterKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeParameter, "nspath.component.rated.voltage")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, constants.RedisParameterDataObjectAliasPrefix+"nspath.component.rated.voltage", parameterKey)
|
||||||
|
|
||||||
|
measurementKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeMeasurement, "nspath.current")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, constants.RedisMeasurementDataObjectAliasPrefix+"nspath.current", measurementKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDataObjectRedisAliasKeyRejectsUnsupportedType(t *testing.T) {
|
||||||
|
_, err := DataObjectRedisAliasKey(constants.DataObjectType("unknown"), "token")
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
@ -17,8 +17,9 @@ import (
|
||||||
const measurementDataObjectPipelineSize = 500
|
const measurementDataObjectPipelineSize = 500
|
||||||
|
|
||||||
type measurementDataObjectHash struct {
|
type measurementDataObjectHash struct {
|
||||||
Key string
|
Key string
|
||||||
Fields map[string]any
|
Aliases []string
|
||||||
|
Fields map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
// MeasurementInitializationRecord contains a measurement and the hierarchy
|
// MeasurementInitializationRecord contains a measurement and the hierarchy
|
||||||
|
|
@ -41,8 +42,8 @@ type MeasurementInitializationRecord struct {
|
||||||
MeasurementBinding orm.JSONMap `gorm:"column:measurement_binding"`
|
MeasurementBinding orm.JSONMap `gorm:"column:measurement_binding"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// InitializeMeasurementDataObjects creates the seven-part, four-part, and
|
// InitializeMeasurementDataObjects creates one seven-part Redis hash and
|
||||||
// token4.token7 Redis hashes for measurements loaded from PostgreSQL.
|
// aliases for all supported measurement token forms.
|
||||||
func InitializeMeasurementDataObjects(ctx context.Context, records []MeasurementInitializationRecord) error {
|
func InitializeMeasurementDataObjects(ctx context.Context, records []MeasurementInitializationRecord) error {
|
||||||
hashes, err := buildMeasurementDataObjectHashes(records)
|
hashes, err := buildMeasurementDataObjectHashes(records)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -60,7 +61,7 @@ func InitializeMeasurementDataObjects(ctx context.Context, records []Measurement
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord) ([]measurementDataObjectHash, error) {
|
func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord) ([]measurementDataObjectHash, error) {
|
||||||
hashes := make([]measurementDataObjectHash, 0, len(records)*3)
|
hashes := make([]measurementDataObjectHash, 0, len(records))
|
||||||
seenKeys := make(map[string]string, len(records)*3)
|
seenKeys := make(map[string]string, len(records)*3)
|
||||||
for _, record := range records {
|
for _, record := range records {
|
||||||
if record.MeasurementMode != constants.MeasurementModeManual &&
|
if record.MeasurementMode != constants.MeasurementModeManual &&
|
||||||
|
|
@ -103,7 +104,8 @@ func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord)
|
||||||
}, ".")
|
}, ".")
|
||||||
twoPartToken := record.ComponentNSPath + "." + record.MeasurementTag
|
twoPartToken := record.ComponentNSPath + "." + record.MeasurementTag
|
||||||
|
|
||||||
for _, token := range []string{fullToken, fourPartToken, twoPartToken} {
|
aliases := []string{fullToken, fourPartToken, twoPartToken}
|
||||||
|
for _, token := range aliases {
|
||||||
if err := validateInitializedMeasurementToken(token); err != nil {
|
if err := validateInitializedMeasurementToken(token); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -139,11 +141,18 @@ func buildMeasurementDataObjectHashes(records []MeasurementInitializationRecord)
|
||||||
"binding": binding,
|
"binding": binding,
|
||||||
}
|
}
|
||||||
owner := fmt.Sprintf("%d/%s", record.MeasurementID, record.ComponentUUID)
|
owner := fmt.Sprintf("%d/%s", record.MeasurementID, record.ComponentUUID)
|
||||||
for _, token := range []string{fullToken, fourPartToken, twoPartToken} {
|
for _, alias := range aliases {
|
||||||
if err := appendMeasurementDataObjectHash(&hashes, seenKeys, token, owner, fields); err != nil {
|
if existingOwner, exists := seenKeys[alias]; exists {
|
||||||
return nil, err
|
return nil, fmt.Errorf(
|
||||||
|
"ambiguous measurement token %q is produced by %q and %q",
|
||||||
|
alias,
|
||||||
|
existingOwner,
|
||||||
|
owner,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
seenKeys[alias] = owner
|
||||||
}
|
}
|
||||||
|
hashes = append(hashes, measurementDataObjectHash{Key: fullToken, Aliases: aliases, Fields: fields})
|
||||||
}
|
}
|
||||||
return hashes, nil
|
return hashes, nil
|
||||||
}
|
}
|
||||||
|
|
@ -159,26 +168,6 @@ func validateInitializedMeasurementToken(token string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func appendMeasurementDataObjectHash(
|
|
||||||
hashes *[]measurementDataObjectHash,
|
|
||||||
seenKeys map[string]string,
|
|
||||||
key string,
|
|
||||||
owner string,
|
|
||||||
fields map[string]any,
|
|
||||||
) error {
|
|
||||||
if existingOwner, exists := seenKeys[key]; exists {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"ambiguous measurement token %q is produced by %q and %q",
|
|
||||||
key,
|
|
||||||
existingOwner,
|
|
||||||
owner,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
seenKeys[key] = owner
|
|
||||||
*hashes = append(*hashes, measurementDataObjectHash{Key: key, Fields: fields})
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func measurementInitializationJSON(value orm.JSONMap) (string, error) {
|
func measurementInitializationJSON(value orm.JSONMap) (string, error) {
|
||||||
encoded, err := json.Marshal(value)
|
encoded, err := json.Marshal(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -200,20 +189,38 @@ func storeMeasurementDataObjectHashes(
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("query previously initialized measurement keys: %w", err)
|
return fmt.Errorf("query previously initialized measurement keys: %w", err)
|
||||||
}
|
}
|
||||||
|
oldAliasKeys, err := rdb.SMembers(ctx, constants.RedisMeasurementDataObjectAliasKeySet).Result()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("query previously initialized measurement alias keys: %w", err)
|
||||||
|
}
|
||||||
currentKeys := make(map[string]struct{}, len(hashes))
|
currentKeys := make(map[string]struct{}, len(hashes))
|
||||||
|
currentAliasKeys := make(map[string]struct{}, len(hashes)*3)
|
||||||
for start := 0; start < len(hashes); start += measurementDataObjectPipelineSize {
|
for start := 0; start < len(hashes); start += measurementDataObjectPipelineSize {
|
||||||
end := min(start+measurementDataObjectPipelineSize, len(hashes))
|
end := min(start+measurementDataObjectPipelineSize, len(hashes))
|
||||||
pipeline := rdb.TxPipeline()
|
pipeline := rdb.TxPipeline()
|
||||||
keyMembers := make([]any, 0, end-start)
|
keyMembers := make([]any, 0, end-start)
|
||||||
|
aliasKeyMembers := make([]any, 0, (end-start)*3)
|
||||||
for _, hash := range hashes[start:end] {
|
for _, hash := range hashes[start:end] {
|
||||||
pipeline.Del(ctx, hash.Key)
|
pipeline.Del(ctx, hash.Key)
|
||||||
pipeline.HSet(ctx, hash.Key, hash.Fields)
|
pipeline.HSet(ctx, hash.Key, hash.Fields)
|
||||||
keyMembers = append(keyMembers, hash.Key)
|
keyMembers = append(keyMembers, hash.Key)
|
||||||
currentKeys[hash.Key] = struct{}{}
|
currentKeys[hash.Key] = struct{}{}
|
||||||
|
for _, alias := range hash.Aliases {
|
||||||
|
aliasKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeMeasurement, alias)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pipeline.Set(ctx, aliasKey, hash.Key, 0)
|
||||||
|
aliasKeyMembers = append(aliasKeyMembers, aliasKey)
|
||||||
|
currentAliasKeys[aliasKey] = struct{}{}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if len(keyMembers) > 0 {
|
if len(keyMembers) > 0 {
|
||||||
pipeline.SAdd(ctx, constants.RedisMeasurementDataObjectKeySet, keyMembers...)
|
pipeline.SAdd(ctx, constants.RedisMeasurementDataObjectKeySet, keyMembers...)
|
||||||
}
|
}
|
||||||
|
if len(aliasKeyMembers) > 0 {
|
||||||
|
pipeline.SAdd(ctx, constants.RedisMeasurementDataObjectAliasKeySet, aliasKeyMembers...)
|
||||||
|
}
|
||||||
if _, err := pipeline.Exec(ctx); err != nil {
|
if _, err := pipeline.Exec(ctx); err != nil {
|
||||||
return fmt.Errorf("write measurement data-object hash batch starting at %d: %w", start, err)
|
return fmt.Errorf("write measurement data-object hash batch starting at %d: %w", start, err)
|
||||||
}
|
}
|
||||||
|
|
@ -235,8 +242,24 @@ func storeMeasurementDataObjectHashes(
|
||||||
}
|
}
|
||||||
cleanupPipeline.SRem(ctx, constants.RedisMeasurementDataObjectKeySet, members...)
|
cleanupPipeline.SRem(ctx, constants.RedisMeasurementDataObjectKeySet, members...)
|
||||||
}
|
}
|
||||||
|
staleAliasKeys := make([]string, 0)
|
||||||
|
for _, aliasKey := range oldAliasKeys {
|
||||||
|
if _, exists := currentAliasKeys[aliasKey]; !exists {
|
||||||
|
staleAliasKeys = append(staleAliasKeys, aliasKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for start := 0; start < len(staleAliasKeys); start += measurementDataObjectPipelineSize {
|
||||||
|
end := min(start+measurementDataObjectPipelineSize, len(staleAliasKeys))
|
||||||
|
cleanupPipeline.Del(ctx, staleAliasKeys[start:end]...)
|
||||||
|
members := make([]any, 0, end-start)
|
||||||
|
for _, aliasKey := range staleAliasKeys[start:end] {
|
||||||
|
members = append(members, aliasKey)
|
||||||
|
}
|
||||||
|
cleanupPipeline.SRem(ctx, constants.RedisMeasurementDataObjectAliasKeySet, members...)
|
||||||
|
}
|
||||||
if len(hashes) == 0 {
|
if len(hashes) == 0 {
|
||||||
cleanupPipeline.Del(ctx, constants.RedisMeasurementDataObjectKeySet)
|
cleanupPipeline.Del(ctx, constants.RedisMeasurementDataObjectKeySet)
|
||||||
|
cleanupPipeline.Del(ctx, constants.RedisMeasurementDataObjectAliasKeySet)
|
||||||
}
|
}
|
||||||
if _, err := cleanupPipeline.Exec(ctx); err != nil {
|
if _, err := cleanupPipeline.Exec(ctx); err != nil {
|
||||||
return fmt.Errorf("remove stale measurement data-object hashes: %w", err)
|
return fmt.Errorf("remove stale measurement data-object hashes: %w", err)
|
||||||
|
|
|
||||||
|
|
@ -9,19 +9,18 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuildMeasurementDataObjectHashesCreatesAllTokenForms(t *testing.T) {
|
func TestBuildMeasurementDataObjectHashesCreatesCanonicalHashAndAllAliases(t *testing.T) {
|
||||||
record := measurementInitializationRecordForTest()
|
record := measurementInitializationRecordForTest()
|
||||||
|
|
||||||
hashes, err := buildMeasurementDataObjectHashes([]MeasurementInitializationRecord{record})
|
hashes, err := buildMeasurementDataObjectHashes([]MeasurementInitializationRecord{record})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, hashes, 3)
|
require.Len(t, hashes, 1)
|
||||||
|
|
||||||
fullToken := "grid000.zone000.station000.220kV_xuefulu1.CTA.bay.IA_rms"
|
fullToken := "grid000.zone000.station000.220kV_xuefulu1.CTA.bay.IA_rms"
|
||||||
fourPartToken := "220kV_xuefulu1.CTA.bay.IA_rms"
|
fourPartToken := "220kV_xuefulu1.CTA.bay.IA_rms"
|
||||||
twoPartToken := "220kV_xuefulu1.IA_rms"
|
twoPartToken := "220kV_xuefulu1.IA_rms"
|
||||||
assert.Equal(t, fullToken, hashes[0].Key)
|
assert.Equal(t, fullToken, hashes[0].Key)
|
||||||
assert.Equal(t, fourPartToken, hashes[1].Key)
|
assert.Equal(t, []string{fullToken, fourPartToken, twoPartToken}, hashes[0].Aliases)
|
||||||
assert.Equal(t, twoPartToken, hashes[2].Key)
|
|
||||||
|
|
||||||
fields := hashes[0].Fields
|
fields := hashes[0].Fields
|
||||||
assert.NotContains(t, fields, "value")
|
assert.NotContains(t, fields, "value")
|
||||||
|
|
@ -35,8 +34,6 @@ func TestBuildMeasurementDataObjectHashesCreatesAllTokenForms(t *testing.T) {
|
||||||
assert.Equal(t, `{"io_address":{"channel":"TM1","device":"CTA","dtype":1,"option":"rms","station":"001"},"type":1}`, fields["data_source"])
|
assert.Equal(t, `{"io_address":{"channel":"TM1","device":"CTA","dtype":1,"option":"rms","station":"001"},"type":1}`, fields["data_source"])
|
||||||
assert.Equal(t, `{}`, fields["event_plan"])
|
assert.Equal(t, `{}`, fields["event_plan"])
|
||||||
assert.Equal(t, `{"ct":{"index":0,"polarity":1,"ratio":1250}}`, fields["binding"])
|
assert.Equal(t, `{"ct":{"index":0,"polarity":1,"ratio":1250}}`, fields["binding"])
|
||||||
assert.Equal(t, fields, hashes[1].Fields)
|
|
||||||
assert.Equal(t, fields, hashes[2].Fields)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildMeasurementDataObjectHashesRejectsAmbiguousShortToken(t *testing.T) {
|
func TestBuildMeasurementDataObjectHashesRejectsAmbiguousShortToken(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,9 @@ import (
|
||||||
const parameterDataObjectPipelineSize = 500
|
const parameterDataObjectPipelineSize = 500
|
||||||
|
|
||||||
type parameterDataObjectHash struct {
|
type parameterDataObjectHash struct {
|
||||||
Key string
|
Key string
|
||||||
Fields map[string]any
|
Aliases []string
|
||||||
|
Fields map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParameterInitializationRecord contains one parameter attribute together with
|
// ParameterInitializationRecord contains one parameter attribute together with
|
||||||
|
|
@ -41,8 +42,8 @@ type ParameterInitializationRecord struct {
|
||||||
DynamicRecordCount int64 `gorm:"column:dynamic_record_count"`
|
DynamicRecordCount int64 `gorm:"column:dynamic_record_count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// InitializeParameterDataObjects creates full and local-short Redis hashes for
|
// InitializeParameterDataObjects creates one full-token Redis hash and full or
|
||||||
// parameter attributes previously loaded from PostgreSQL.
|
// local-short token aliases for each parameter loaded from PostgreSQL.
|
||||||
func InitializeParameterDataObjects(ctx context.Context, records []ParameterInitializationRecord) error {
|
func InitializeParameterDataObjects(ctx context.Context, records []ParameterInitializationRecord) error {
|
||||||
hashes, err := buildParameterDataObjectHashes(records)
|
hashes, err := buildParameterDataObjectHashes(records)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -60,7 +61,7 @@ func InitializeParameterDataObjects(ctx context.Context, records []ParameterInit
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]parameterDataObjectHash, error) {
|
func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]parameterDataObjectHash, error) {
|
||||||
hashes := make([]parameterDataObjectHash, 0, len(records)*2)
|
hashes := make([]parameterDataObjectHash, 0, len(records))
|
||||||
seenKeys := make(map[string]string, len(records)*2)
|
seenKeys := make(map[string]string, len(records)*2)
|
||||||
|
|
||||||
for _, record := range records {
|
for _, record := range records {
|
||||||
|
|
@ -91,8 +92,9 @@ func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]
|
||||||
record.AttributeName,
|
record.AttributeName,
|
||||||
}, ".")
|
}, ".")
|
||||||
|
|
||||||
if err := validateInitializedParameterToken(fullToken); err != nil {
|
aliases := []string{fullToken}
|
||||||
return nil, err
|
if record.StationIsLocal {
|
||||||
|
aliases = append(aliases, shortToken)
|
||||||
}
|
}
|
||||||
fields := map[string]any{
|
fields := map[string]any{
|
||||||
"value": value,
|
"value": value,
|
||||||
|
|
@ -107,19 +109,21 @@ func buildParameterDataObjectHashes(records []ParameterInitializationRecord) ([]
|
||||||
record.AttributeGroup,
|
record.AttributeGroup,
|
||||||
record.AttributeName,
|
record.AttributeName,
|
||||||
}, "/")
|
}, "/")
|
||||||
if err := appendParameterDataObjectHash(&hashes, seenKeys, fullToken, owner, fields); err != nil {
|
for _, alias := range aliases {
|
||||||
return nil, err
|
if err := validateInitializedParameterToken(alias); err != nil {
|
||||||
}
|
return nil, err
|
||||||
|
}
|
||||||
if !record.StationIsLocal {
|
if existingOwner, exists := seenKeys[alias]; exists {
|
||||||
continue
|
return nil, fmt.Errorf(
|
||||||
}
|
"ambiguous parameter token %q is produced by %q and %q",
|
||||||
if err := validateInitializedParameterToken(shortToken); err != nil {
|
alias,
|
||||||
return nil, err
|
existingOwner,
|
||||||
}
|
owner,
|
||||||
if err := appendParameterDataObjectHash(&hashes, seenKeys, shortToken, owner, fields); err != nil {
|
)
|
||||||
return nil, err
|
}
|
||||||
|
seenKeys[alias] = owner
|
||||||
}
|
}
|
||||||
|
hashes = append(hashes, parameterDataObjectHash{Key: fullToken, Aliases: aliases, Fields: fields})
|
||||||
}
|
}
|
||||||
return hashes, nil
|
return hashes, nil
|
||||||
}
|
}
|
||||||
|
|
@ -135,26 +139,6 @@ func validateInitializedParameterToken(token string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func appendParameterDataObjectHash(
|
|
||||||
hashes *[]parameterDataObjectHash,
|
|
||||||
seenKeys map[string]string,
|
|
||||||
key string,
|
|
||||||
owner string,
|
|
||||||
fields map[string]any,
|
|
||||||
) error {
|
|
||||||
if existingOwner, exists := seenKeys[key]; exists {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"ambiguous parameter token %q is produced by %q and %q",
|
|
||||||
key,
|
|
||||||
existingOwner,
|
|
||||||
owner,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
seenKeys[key] = owner
|
|
||||||
*hashes = append(*hashes, parameterDataObjectHash{Key: key, Fields: fields})
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func parameterRedisValue(rawJSON string) (any, error) {
|
func parameterRedisValue(rawJSON string) (any, error) {
|
||||||
decoder := json.NewDecoder(bytes.NewBufferString(rawJSON))
|
decoder := json.NewDecoder(bytes.NewBufferString(rawJSON))
|
||||||
decoder.UseNumber()
|
decoder.UseNumber()
|
||||||
|
|
@ -194,30 +178,86 @@ func storeParameterDataObjectHashes(
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("query previously initialized parameter keys: %w", err)
|
return fmt.Errorf("query previously initialized parameter keys: %w", err)
|
||||||
}
|
}
|
||||||
cleanupPipeline := rdb.TxPipeline()
|
oldAliasKeys, err := rdb.SMembers(ctx, constants.RedisParameterDataObjectAliasKeySet).Result()
|
||||||
for start := 0; start < len(oldKeys); start += parameterDataObjectPipelineSize {
|
if err != nil {
|
||||||
end := min(start+parameterDataObjectPipelineSize, len(oldKeys))
|
return fmt.Errorf("query previously initialized parameter alias keys: %w", err)
|
||||||
cleanupPipeline.Del(ctx, oldKeys[start:end]...)
|
|
||||||
}
|
|
||||||
cleanupPipeline.Del(ctx, constants.RedisParameterDataObjectKeySet)
|
|
||||||
if _, err := cleanupPipeline.Exec(ctx); err != nil {
|
|
||||||
return fmt.Errorf("remove stale parameter data-object hashes: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
currentKeys := make(map[string]struct{}, len(hashes))
|
||||||
|
currentAliasKeys := make(map[string]struct{}, len(hashes)*2)
|
||||||
for start := 0; start < len(hashes); start += parameterDataObjectPipelineSize {
|
for start := 0; start < len(hashes); start += parameterDataObjectPipelineSize {
|
||||||
end := min(start+parameterDataObjectPipelineSize, len(hashes))
|
end := min(start+parameterDataObjectPipelineSize, len(hashes))
|
||||||
pipeline := rdb.TxPipeline()
|
pipeline := rdb.TxPipeline()
|
||||||
keyMembers := make([]any, 0, end-start)
|
keyMembers := make([]any, 0, end-start)
|
||||||
|
aliasKeyMembers := make([]any, 0, (end-start)*2)
|
||||||
for _, hash := range hashes[start:end] {
|
for _, hash := range hashes[start:end] {
|
||||||
|
pipeline.Del(ctx, hash.Key)
|
||||||
pipeline.HSet(ctx, hash.Key, hash.Fields)
|
pipeline.HSet(ctx, hash.Key, hash.Fields)
|
||||||
keyMembers = append(keyMembers, hash.Key)
|
keyMembers = append(keyMembers, hash.Key)
|
||||||
|
currentKeys[hash.Key] = struct{}{}
|
||||||
|
for _, alias := range hash.Aliases {
|
||||||
|
aliasKey, err := DataObjectRedisAliasKey(constants.DataObjectTypeParameter, alias)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pipeline.Set(ctx, aliasKey, hash.Key, 0)
|
||||||
|
aliasKeyMembers = append(aliasKeyMembers, aliasKey)
|
||||||
|
currentAliasKeys[aliasKey] = struct{}{}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if len(keyMembers) > 0 {
|
if len(keyMembers) > 0 {
|
||||||
pipeline.SAdd(ctx, constants.RedisParameterDataObjectKeySet, keyMembers...)
|
pipeline.SAdd(ctx, constants.RedisParameterDataObjectKeySet, keyMembers...)
|
||||||
}
|
}
|
||||||
|
if len(aliasKeyMembers) > 0 {
|
||||||
|
pipeline.SAdd(ctx, constants.RedisParameterDataObjectAliasKeySet, aliasKeyMembers...)
|
||||||
|
}
|
||||||
if _, err := pipeline.Exec(ctx); err != nil {
|
if _, err := pipeline.Exec(ctx); err != nil {
|
||||||
return fmt.Errorf("write parameter data-object hash batch starting at %d: %w", start, err)
|
return fmt.Errorf("write parameter data-object hash batch starting at %d: %w", start, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := cleanupStaleParameterDataObjectKeys(
|
||||||
|
ctx,
|
||||||
|
rdb,
|
||||||
|
oldKeys,
|
||||||
|
oldAliasKeys,
|
||||||
|
currentKeys,
|
||||||
|
currentAliasKeys,
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanupStaleParameterDataObjectKeys(
|
||||||
|
ctx context.Context,
|
||||||
|
rdb *redis.Client,
|
||||||
|
oldKeys, oldAliasKeys []string,
|
||||||
|
currentKeys, currentAliasKeys map[string]struct{},
|
||||||
|
) error {
|
||||||
|
pipeline := rdb.TxPipeline()
|
||||||
|
for _, key := range oldKeys {
|
||||||
|
if _, exists := currentKeys[key]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pipeline.Del(ctx, key)
|
||||||
|
pipeline.SRem(ctx, constants.RedisParameterDataObjectKeySet, key)
|
||||||
|
}
|
||||||
|
for _, aliasKey := range oldAliasKeys {
|
||||||
|
if _, exists := currentAliasKeys[aliasKey]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pipeline.Del(ctx, aliasKey)
|
||||||
|
pipeline.SRem(ctx, constants.RedisParameterDataObjectAliasKeySet, aliasKey)
|
||||||
|
}
|
||||||
|
if len(currentKeys) == 0 {
|
||||||
|
pipeline.Del(ctx, constants.RedisParameterDataObjectKeySet)
|
||||||
|
}
|
||||||
|
if len(currentAliasKeys) == 0 {
|
||||||
|
pipeline.Del(ctx, constants.RedisParameterDataObjectAliasKeySet)
|
||||||
|
}
|
||||||
|
if _, err := pipeline.Exec(ctx); err != nil {
|
||||||
|
return fmt.Errorf("remove stale parameter data-object keys: %w", err)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuildParameterDataObjectHashesCreatesFullAndLocalKeys(t *testing.T) {
|
func TestBuildParameterDataObjectHashesCreatesCanonicalHashAndLocalAliases(t *testing.T) {
|
||||||
records := []ParameterInitializationRecord{
|
records := []ParameterInitializationRecord{
|
||||||
{
|
{
|
||||||
GridTag: "grid000",
|
GridTag: "grid000",
|
||||||
|
|
@ -30,19 +30,18 @@ func TestBuildParameterDataObjectHashesCreatesFullAndLocalKeys(t *testing.T) {
|
||||||
|
|
||||||
hashes, err := buildParameterDataObjectHashes(records)
|
hashes, err := buildParameterDataObjectHashes(records)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, hashes, 2)
|
require.Len(t, hashes, 1)
|
||||||
|
|
||||||
fullToken := "grid000.zone000.station000.220kV_xuefulu1.cable_22.base_extend.vnom_kv"
|
fullToken := "grid000.zone000.station000.220kV_xuefulu1.cable_22.base_extend.vnom_kv"
|
||||||
shortToken := "220kV_xuefulu1.cable_22.base_extend.vnom_kv"
|
shortToken := "220kV_xuefulu1.cable_22.base_extend.vnom_kv"
|
||||||
assert.Equal(t, fullToken, hashes[0].Key)
|
assert.Equal(t, fullToken, hashes[0].Key)
|
||||||
assert.Equal(t, shortToken, hashes[1].Key)
|
assert.Equal(t, []string{fullToken, shortToken}, hashes[0].Aliases)
|
||||||
assert.Equal(t, "7800.00", hashes[0].Fields["value"])
|
assert.Equal(t, "7800.00", hashes[0].Fields["value"])
|
||||||
assert.Equal(t, "PARAM", hashes[0].Fields["meta"])
|
assert.Equal(t, "PARAM", hashes[0].Fields["meta"])
|
||||||
assert.Equal(t, "DOUBLE PRECISION", hashes[0].Fields["type"])
|
assert.Equal(t, "DOUBLE PRECISION", hashes[0].Fields["type"])
|
||||||
assert.Equal(t, shortToken, hashes[0].Fields["name"])
|
assert.Equal(t, shortToken, hashes[0].Fields["name"])
|
||||||
assert.Equal(t, "额定电压", hashes[0].Fields["description"])
|
assert.Equal(t, "额定电压", hashes[0].Fields["description"])
|
||||||
assert.Equal(t, fullToken, hashes[0].Fields["id"])
|
assert.Equal(t, fullToken, hashes[0].Fields["id"])
|
||||||
assert.Equal(t, hashes[0].Fields, hashes[1].Fields)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildParameterDataObjectHashesSkipsShortKeyForNonLocalStation(t *testing.T) {
|
func TestBuildParameterDataObjectHashesSkipsShortKeyForNonLocalStation(t *testing.T) {
|
||||||
|
|
@ -69,6 +68,7 @@ func TestBuildParameterDataObjectHashesSkipsShortKeyForNonLocalStation(t *testin
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, hashes, 1)
|
require.Len(t, hashes, 1)
|
||||||
assert.Equal(t, "grid.zone.station.nspath.component.stable.attribute", hashes[0].Key)
|
assert.Equal(t, "grid.zone.station.nspath.component.stable.attribute", hashes[0].Key)
|
||||||
|
assert.Equal(t, []string{"grid.zone.station.nspath.component.stable.attribute"}, hashes[0].Aliases)
|
||||||
assert.Equal(t, true, hashes[0].Fields["value"])
|
assert.Equal(t, true, hashes[0].Fields["value"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue