optimize the circuit diagram loading function

modelRT/config/model_config.go

@@ -0,0 +1,13 @@
+package config
+
+import (
+	"sync"
+
+	"modelRT/orm"
+)
+
+type ModelParseConfig struct {
+	CircuitDiagramID    int64
+	CircuitDiagramInfos []orm.CircuitDiagram
+	CircuitDiagramMap   *sync.Map
+}

modelRT/constant/electrical_components.go

@@ -0,0 +1,11 @@
+// Package constant define constant value
+package constant
+
+const (
+	// NullableType 空类型类型
+	NullableType = iota
+	// BusbarType 母线类型
+	BusbarType
+	// AsynchronousMotorType 异步电动机类型
+	AsynchronousMotorType
+)

modelRT/database/posgres_operator.go

@@ -3,17 +3,19 @@ package database
 
 import (
 	"context"
-	"encoding/json"
 	"strconv"
+	"sync"
 	"time"
 
+	"modelRT/config"
 	"modelRT/orm"
 
+	"github.com/panjf2000/ants/v2"
 	"go.uber.org/zap"
 )
 
 // LoadCircuitDiagramFromPostgresDB return the result of query circuit diagram info from postgresDB
-func LoadCircuitDiagramFromPostgresDB(ctx context.Context, logger *zap.Logger) ([]orm.CircuitDiagram, error) {
+func LoadCircuitDiagramFromPostgresDB(ctx context.Context, pool *ants.PoolWithFunc, circuitDiagramsMap *sync.Map, logger *zap.Logger) error {
 	var circuitDiagramOverviews []orm.CircuitDiagramOverview
 	// ctx超时判断
 	cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
@@ -21,28 +23,24 @@ func LoadCircuitDiagramFromPostgresDB(ctx context.Context, logger *zap.Logger) (
 	result := _globalPostgresClient.WithContext(cancelCtx).Find(&circuitDiagramOverviews)
 	if result.Error != nil {
 		logger.Error("query circuit diagram overview info failed", zap.Error(result.Error))
+		return result.Error
 	}
 
-	var circuitDiagrams []orm.CircuitDiagram
+	for _, overview := range circuitDiagramOverviews {
-	for _, diagram := range circuitDiagramOverviews {
 		cancelCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
 		defer cancel()
-		var circuitDiagram orm.CircuitDiagram
+		var circuitDiagramInfos []orm.CircuitDiagram
-		tableName := "circuit_diagram_" + strconv.FormatInt(diagram.ID, 10)
+		tableName := "circuit_diagram_" + strconv.FormatInt(overview.ID, 10)
-		result := _globalPostgresClient.Table(tableName).WithContext(cancelCtx).Find(&circuitDiagram)
+		result := _globalPostgresClient.Table(tableName).WithContext(cancelCtx).Order("id asc").Find(&circuitDiagramInfos)
 		if result.Error != nil {
-			logger.Error("query circuit diagram info failed", zap.Error(result.Error))
+			logger.Error("query circuit diagram detail infos failed", zap.Error(result.Error))
-			return nil, result.Error
+			return result.Error
 		}
-		circuitDiagrams = append(circuitDiagrams, circuitDiagram)
+		pool.Invoke(config.ModelParseConfig{
+			CircuitDiagramID:    overview.ID,
+			CircuitDiagramInfos: circuitDiagramInfos,
+			CircuitDiagramMap:   circuitDiagramsMap,
+		})
 	}
-
+	return nil
-	for _, diagram := range circuitDiagrams {
-		// TODO
-		// 根据 diagram 的 type 类型选择不同的承接结构
-		// 换一个解析更快的 json encoder
-		json.Unmarshal([]byte(diagram.OtherParams), nil)
-		// unmarshal diagram
-	}
-	return circuitDiagrams, nil
 }

modelRT/go.mod

@@ -4,7 +4,10 @@ go 1.22.5
 
 require (
 	github.com/gin-gonic/gin v1.10.0
+	github.com/gofrs/uuid v4.4.0+incompatible
+	github.com/json-iterator/go v1.1.12
 	github.com/natefinch/lumberjack v2.0.0+incompatible
+	github.com/panjf2000/ants/v2 v2.10.0
 	github.com/spf13/viper v1.19.0
 	go.uber.org/zap v1.27.0
 	gorm.io/driver/postgres v1.5.9
@@ -31,7 +34,6 @@ require (
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect

modelRT/go.sum

@@ -32,6 +32,8 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx
 github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
+github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -74,6 +76,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
+github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8=
+github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -104,6 +108,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
@@ -128,6 +133,7 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjs
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

modelRT/main.go

@@ -4,6 +4,7 @@ package main
 import (
 	"context"
 	"flag"
+	"sync"
 	"time"
 
 	"modelRT/config"
@@ -11,8 +12,10 @@ import (
 	"modelRT/handler"
 	"modelRT/log"
 	"modelRT/middleware"
+	"modelRT/model"
 
 	"github.com/gin-gonic/gin"
+	"github.com/panjf2000/ants/v2"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
 )
@@ -30,9 +33,10 @@ var (
 )
 
 var (
-	modelRTConfig    config.ModelRTConfig
+	modelRTConfig     config.ModelRTConfig
-	postgresDBClient *gorm.DB
+	postgresDBClient  *gorm.DB
-	logger           *zap.Logger
+	logger            *zap.Logger
+	circuitDiagramMap sync.Map
 )
 
 // TODO 使用 wire 依赖注入
@@ -41,7 +45,6 @@ func main() {
 	ctx := context.TODO()
 
 	modelRTConfig = config.ReadAndInitConfig(*modelRTConfigDir, *modelRTConfigName, *modelRTConfigType)
-	// TODO 创建 Redis 与 pg client
 	// init postgresDBClient
 	postgresDBClient = database.GetPostgresDBInstance(ctx, modelRTConfig.PostgresDBURI)
 
@@ -56,10 +59,21 @@ func main() {
 	// init logger
 	logger = log.GetLoggerInstance(modelRTConfig.LCfg)
 	defer logger.Sync()
+	// init ants pool
+	pool, err := ants.NewPoolWithFunc(modelRTConfig.ParseConcurrentQuantity, model.ParseFunc)
+	if err != nil {
+		logger.Error("init concurrent parse task pool failed", zap.Error(err))
+		panic(err)
+	}
+	defer ants.Release()
 
 	// load circuit diagram from postgres
-	database.LoadCircuitDiagramFromPostgresDB(ctx, logger)
+	err = database.LoadCircuitDiagramFromPostgresDB(ctx, pool, &circuitDiagramMap, logger)
-	// TODO 从 pg 加载到 redis
+	if err != nil {
+		logger.Error("load circuit diagrams from postgres failed", zap.Error(err))
+		panic(err)
+	}
+
 	engine := gin.Default()
 	engine.Use(limiter.Middleware)
 	engine.GET("/model/model_load", handler.ModelLoad)

modelRT/model/busbar_section.go

@@ -1,9 +1,12 @@
 package model
 
+import "github.com/gofrs/uuid"
+
 type BusbarSection struct {
 	// 母线基本参数
-	Name               string  // 母线端名称,默认值BusX
+	Name               string // 母线端名称,默认值BusX
-	BusbarNumber       int     // 母线编号,默认值1
+	BusbarNumber       int    // 母线编号,默认值1
+	UUID               uuid.UUID
 	StandardVoltage    float32 // 标准电压,单位kV,范围值在000.01~500.00
 	Desc               string  // 描述
 	IsService          bool    // 是否服役,值为运行/退出
@@ -58,10 +61,15 @@ type BusbarSection struct {
 	StatusMeasurementPoint  []string // 状态测点测点
 }
 
-func NewBusbarSection(name string) *BusbarSection {
+func NewBusbarSection(name string) (*BusbarSection, error) {
+	uuid, err := uuid.NewV4()
+	if err != nil {
+		return nil, err
+	}
 	return &BusbarSection{
 		Name: name,
-	}
+		UUID: uuid,
+	}, nil
 }
 
 func (b *BusbarSection) BusNameLenCheck() bool {

modelRT/model/model_parse.go

@@ -0,0 +1,33 @@
+package model
+
+import (
+	"modelRT/config"
+
+	jsoniter "github.com/json-iterator/go"
+
+	"go.uber.org/zap"
+)
+
+var ParseFunc = func(parseConfig interface{}) {
+	logger := zap.L()
+
+	modelParseConfig, ok := parseConfig.(config.ModelParseConfig)
+	if !ok {
+		logger.Error("conversion model parse config type failed")
+		return
+	}
+
+	for _, circuitDiagram := range modelParseConfig.CircuitDiagramInfos {
+		parseStruct := SelectModelByType(circuitDiagram.Type)
+		err := jsoniter.Unmarshal([]byte(circuitDiagram.OtherParams), parseStruct)
+		if err != nil {
+			logger.Error("parsing circuit diagram params failed", zap.Int64("circuit_diagram_id", circuitDiagram.ID), zap.Error(err))
+			panic(err)
+		}
+		// TODO 利用多叉树结构存储解析后的电路图
+	}
+	key := modelParseConfig.CircuitDiagramID
+	// TODO 形成 key-多叉树结构
+	modelParseConfig.CircuitDiagramMap.Store(key, nil)
+	return
+}

modelRT/model/model_select.go

@@ -0,0 +1,10 @@
+package model
+
+import "modelRT/constant"
+
+func SelectModelByType(modelType int) any {
+	if modelType == constant.BusbarType {
+		return BusbarSection{}
+	}
+	return nil
+}

modelRT/orm/circuit_diagram.go

@@ -6,7 +6,8 @@ import "time"
 type CircuitDiagram struct {
 	ID          int64 `gorm:""`
 	ParentID    int64
-	Type        string
+	Type        int
+	UUID        string
 	OtherParams string
 	CreatedTime time.Time
 	UpdateTime  time.Time