feat:add log config &concurrent parse config
This commit is contained in:
douxu
parent
436677f2f8
commit
bc06dedfc0
|
|
@ -0,0 +1,86 @@
|
|||
package comtrade
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"wave_record/config"
|
||||
"wave_record/constant"
|
||||
"wave_record/util"
|
||||
|
||||
"github.com/yonwoo9/go-comtrade"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var ParseFunc = func(parseConfig interface{}) {
|
||||
logger := zap.L()
|
||||
comtradeConfig, ok := parseConfig.(config.ComtradeFileConfig)
|
||||
if !ok {
|
||||
// TODO 增加日志报错中输出文件名
|
||||
logger.Error("conversion parsing comtrade parameter type failed")
|
||||
return
|
||||
}
|
||||
|
||||
parseComtradeData(comtradeConfig.ConfigFilePath, *&comtradeConfig.DatafilePath)
|
||||
}
|
||||
|
||||
func ParseComtradeFile(monitorDir string, addChan chan string, comtradeMap *sync.Map) {
|
||||
logger := zap.L()
|
||||
|
||||
for {
|
||||
addFilePath, ok := <-addChan
|
||||
if !ok {
|
||||
logger.Error("monitor add channel closed", zap.Bool("channel_status", ok))
|
||||
return
|
||||
}
|
||||
|
||||
lastElement := filepath.Base(addFilePath)
|
||||
fileName := strings.Split(lastElement, ".")[0]
|
||||
fileExtension := filepath.Ext(lastElement)
|
||||
logger.Info("add comtrade file info", zap.String("file_path", addFilePath),
|
||||
zap.String("file_name", fileName), zap.String("file_extension", fileExtension))
|
||||
|
||||
switch fileExtension {
|
||||
case constant.ConfigFileSuffix:
|
||||
configFilePath := addFilePath
|
||||
|
||||
dataFilePath, exist := comtradeMap.Load(addFilePath)
|
||||
if exist {
|
||||
if dataFilePath == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
go parseComtradeData(configFilePath, dataFilePath.(string))
|
||||
} else {
|
||||
comtradeMap.Store(configFilePath, "")
|
||||
}
|
||||
case constant.DataFileSuffix:
|
||||
configFileName := strings.Join([]string{fileName, constant.ConfigFileSuffix}, "")
|
||||
configFilePath := filepath.Join(monitorDir, configFileName)
|
||||
logger.Info("config path of comtrade file", zap.String("file_path", configFilePath))
|
||||
exist := util.FileExists(configFilePath)
|
||||
if exist {
|
||||
comtradeMap.Store(configFilePath, addFilePath)
|
||||
addChan <- configFilePath
|
||||
continue
|
||||
}
|
||||
addChan <- addFilePath
|
||||
default:
|
||||
logger.Warn("no support file style", zap.String("file_style", fileExtension))
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseComtradeData(configFilePath, dataFilePath string) (*comtrade.Comtrade, error) {
|
||||
logger := zap.L()
|
||||
|
||||
comtrade, err := comtrade.ParseComtrade(configFilePath, dataFilePath)
|
||||
if err != nil {
|
||||
logger.Error("parse comtrade file failed", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
return comtrade, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
// Package config define config struct of wave record project
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"wave_record/log"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// WaveRecordConfig is designed for wave record config struct
|
||||
type WaveRecordConfig struct {
|
||||
MonitorDir string
|
||||
ParseConcurrentQuantity int // parse comtrade file concurrent quantity
|
||||
MongoDBURI string
|
||||
MongoDBDataBase string
|
||||
LCfg log.LogConfig // log config
|
||||
}
|
||||
|
||||
// ComtradeFileConfig define config struct of parse comtrade data
|
||||
type ComtradeFileConfig struct {
|
||||
ConfigFilePath string
|
||||
DatafilePath string
|
||||
}
|
||||
|
||||
// ReadAndInitConfig return wave record project config struct
|
||||
func ReadAndInitConfig(configDir, configName, configType string) (waveRecordConfig WaveRecordConfig) {
|
||||
config := viper.New()
|
||||
config.AddConfigPath(configDir)
|
||||
config.SetConfigName(configName)
|
||||
config.SetConfigType(configType)
|
||||
if err := config.ReadInConfig(); err != nil {
|
||||
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
|
||||
panic("can not find conifg file")
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
|
||||
waveRecordConfig.MonitorDir = config.GetString("comtrade_monitor_dir")
|
||||
// init mongodb config from config.yaml
|
||||
mongoDBHost := config.GetString("mongodb_host")
|
||||
mongoDBPort := config.GetString("mongodb_port")
|
||||
waveRecordConfig.MongoDBURI = strings.Join([]string{"mongodb://", mongoDBHost, ":", mongoDBPort}, "")
|
||||
waveRecordConfig.MongoDBDataBase = config.GetString("mongodb_database")
|
||||
// init zap log config from config.yaml
|
||||
waveRecordConfig.LCfg.Level = config.GetString("log_level")
|
||||
waveRecordConfig.LCfg.FileName = config.GetString("log_filename")
|
||||
waveRecordConfig.LCfg.MaxSize = config.GetInt("log_maxsize")
|
||||
waveRecordConfig.LCfg.MaxBackups = config.GetInt("log_maxbackups")
|
||||
waveRecordConfig.LCfg.MaxAge = config.GetInt("log_maxage")
|
||||
|
||||
waveRecordConfig.ParseConcurrentQuantity = config.GetInt("parse_concurrent_quantity")
|
||||
|
||||
return waveRecordConfig
|
||||
}
|
||||
|
|
@ -1 +1,13 @@
|
|||
comtrade_file_dir: /tmp
|
||||
comtrade_monitor_dir: "/tmp"
|
||||
|
||||
mongodb_host: "localhost"
|
||||
mongodb_port: "27017"
|
||||
mongodb_database: "wave_record"
|
||||
|
||||
log_level: "debug"
|
||||
log_filename: "/log/wave_record.log"
|
||||
log_maxsize: 1
|
||||
log_maxbackups: 5
|
||||
log_maxage: 30
|
||||
|
||||
parse_concurrent_quantity: 10
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
var (
|
||||
client *mongo.Client
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// GetMongoDBInstance return instance of mongoDB client
|
||||
func GetMongoDBInstance(ctx context.Context, mongoDBURI string) *mongo.Client {
|
||||
once.Do(func() {
|
||||
client = initMongoDBClient(ctx, mongoDBURI)
|
||||
})
|
||||
return client
|
||||
}
|
||||
|
||||
// initMongoDBClient return successfully initialized mongoDB client
|
||||
func initMongoDBClient(ctx context.Context, mongoDBURI string) *mongo.Client {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
mongoDBClient, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoDBURI))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return mongoDBClient
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/yonwoo9/go-comtrade"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// StorageComtradeIntoMongoDB return the result of storing comtrade data into mongoDB
|
||||
func StorageComtradeIntoMongoDB(ctx context.Context, comtradeData *comtrade.Comtrade, dbName string, client *mongo.Client, logger *zap.Logger) error {
|
||||
collection := client.Database(dbName).Collection(comtradeData.Conf.StationName)
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
comtradeBson, err := bson.Marshal(comtradeData)
|
||||
if err != nil {
|
||||
logger.Error("bson marshal comtrade data failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = collection.InsertOne(ctx, comtradeBson)
|
||||
if err != nil {
|
||||
logger.Error("insert comtrade data into mongoDB failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -4,15 +4,22 @@ go 1.22.5
|
|||
|
||||
require (
|
||||
github.com/fsnotify/fsnotify v1.7.0
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible
|
||||
github.com/panjf2000/ants/v2 v2.10.0
|
||||
github.com/spf13/viper v1.19.0
|
||||
github.com/yonwoo9/go-comtrade v0.0.3
|
||||
go.mongodb.org/mongo-driver v1.16.0
|
||||
go.uber.org/zap v1.21.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.4.0 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.17.2 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
|
|
@ -21,11 +28,18 @@ require (
|
|||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
golang.org/x/crypto v0.22.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
golang.org/x/sync v0.7.0 // indirect
|
||||
golang.org/x/sys v0.19.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
|
||||
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -8,10 +10,14 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
|||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
|
|
@ -23,6 +29,12 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V
|
|||
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
|
||||
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
|
||||
github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8=
|
||||
github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
@ -55,14 +67,26 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
|||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/yonwoo9/go-comtrade v0.0.3 h1:+Qbjn26MMXNacsGYdbYh4jNfMWiIdF8vRs6a0q/0xrU=
|
||||
github.com/yonwoo9/go-comtrade v0.0.3/go.mod h1:FhyIfNGvvEfVXkCzzc5Xsp8HBtHK9AQxF0d2jkp5d4g=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4=
|
||||
go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
|
|
@ -75,32 +99,49 @@ go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
|
|||
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
|
||||
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
@ -110,6 +151,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
|
|||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
// Package log define log struct of wave record project
|
||||
package log
|
||||
|
||||
import (
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/natefinch/lumberjack"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
var (
|
||||
logger *zap.Logger
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// LogConfig define log config of wave record project
|
||||
type LogConfig struct {
|
||||
Level string `json:"level"` // Level 最低日志等级，DEBUG<INFO<WARN<ERROR<FATAL INFO-->收集INFO等级以上的日志
|
||||
FileName string `json:"file_name"` // FileName 日志文件位置
|
||||
MaxSize int `json:"max_size"` // MaxSize 进行切割之前,日志文件的最大大小(MB为单位)默认为100MB
|
||||
MaxAge int `json:"max_age"` // MaxAge 是根据文件名中编码的时间戳保留旧日志文件的最大天数。
|
||||
MaxBackups int `json:"max_backups"` // MaxBackups 是要保留的旧日志文件的最大数量。默认是保留所有旧的日志文件(尽管 MaxAge 可能仍会导致它们被删除)
|
||||
}
|
||||
|
||||
// getEncoder responsible for setting the log format for encoding
|
||||
func getEncoder() zapcore.Encoder {
|
||||
encodeConfig := zap.NewProductionEncoderConfig()
|
||||
// serialization time eg:2006-01-02 15:04:05
|
||||
encodeConfig.EncodeTime = zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05")
|
||||
encodeConfig.TimeKey = "time"
|
||||
encodeConfig.EncodeLevel = zapcore.CapitalLevelEncoder
|
||||
encodeConfig.EncodeCaller = zapcore.ShortCallerEncoder
|
||||
return zapcore.NewJSONEncoder(encodeConfig)
|
||||
}
|
||||
|
||||
// getLogWriter responsible for setting the location of log storage
|
||||
func getLogWriter(filename string, maxsize, maxBackup, maxAge int) zapcore.WriteSyncer {
|
||||
lumberJackLogger := &lumberjack.Logger{
|
||||
Filename: filename, // log file position
|
||||
MaxSize: maxsize, // log file maxsize
|
||||
MaxAge: maxAge, // maximum number of day files retained
|
||||
MaxBackups: maxBackup, // maximum number of old files retained
|
||||
Compress: false, // whether to compress
|
||||
}
|
||||
syncFile := zapcore.AddSync(lumberJackLogger)
|
||||
// TODO:增加调试输出到控制台设置，其他模式无控制台数据
|
||||
syncConsole := zapcore.AddSync(os.Stderr)
|
||||
return zapcore.NewMultiWriteSyncer(syncFile, syncConsole)
|
||||
}
|
||||
|
||||
// initLogger return successfully initialized zap logger
|
||||
func initLogger(lCfg LogConfig) *zap.Logger {
|
||||
writeSyncer := getLogWriter(lCfg.FileName, lCfg.MaxSize, lCfg.MaxBackups, lCfg.MaxAge)
|
||||
encoder := getEncoder()
|
||||
|
||||
l := new(zapcore.Level)
|
||||
err := l.UnmarshalText([]byte(lCfg.Level))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
core := zapcore.NewCore(encoder, writeSyncer, l)
|
||||
logger = zap.New(core, zap.AddCaller())
|
||||
zap.ReplaceGlobals(logger)
|
||||
|
||||
return logger
|
||||
}
|
||||
|
||||
// GetLoggerInstance return instance of zap logger
|
||||
func GetLoggerInstance(lCfg LogConfig) *zap.Logger {
|
||||
once.Do(func() {
|
||||
logger = initLogger(lCfg)
|
||||
})
|
||||
return logger
|
||||
}
|
||||
|
|
@ -1,20 +1,21 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"wave_record/constant"
|
||||
"wave_record/comtrade"
|
||||
"wave_record/config"
|
||||
"wave_record/database"
|
||||
"wave_record/log"
|
||||
"wave_record/util"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/yonwoo9/go-comtrade"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/panjf2000/ants/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -24,133 +25,52 @@ var (
|
|||
)
|
||||
|
||||
var (
|
||||
monitorDir string
|
||||
addChannel chan string
|
||||
comtradeMap sync.Map
|
||||
logger *zap.Logger
|
||||
waveRecordConfig config.WaveRecordConfig
|
||||
addChannel chan string
|
||||
comtradeMap sync.Map
|
||||
mongoDBClient *mongo.Client
|
||||
logger *zap.Logger
|
||||
)
|
||||
|
||||
func init() {
|
||||
addChannel = make(chan string, 10)
|
||||
logger, _ = zap.NewProduction()
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
ctx := context.TODO()
|
||||
|
||||
waveRecordConfig = config.ReadAndInitConfig(*comtradeConfigDir, *comtradeConfigName, *comtradeConfigType)
|
||||
|
||||
// init mongoDBClient
|
||||
mongoDBClient = database.GetMongoDBInstance(ctx, waveRecordConfig.MongoDBURI)
|
||||
|
||||
// init logger
|
||||
logger = log.GetLoggerInstance(waveRecordConfig.LCfg)
|
||||
|
||||
defer func() {
|
||||
if err := mongoDBClient.Disconnect(ctx); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
defer logger.Sync()
|
||||
readConfig()
|
||||
logger.Info("comtrade monitor Dir", zap.String("monitor_dir", monitorDir))
|
||||
go dirWatch(monitorDir, addChannel)
|
||||
|
||||
parseComtradeFile(addChannel, comtradeMap)
|
||||
}
|
||||
defer ants.Release()
|
||||
|
||||
func readConfig() {
|
||||
config := viper.New()
|
||||
config.AddConfigPath(*comtradeConfigDir)
|
||||
config.SetConfigName(*comtradeConfigName)
|
||||
config.SetConfigType(*comtradeConfigType)
|
||||
if err := config.ReadInConfig(); err != nil {
|
||||
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
|
||||
panic("can not find conifg file")
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
// get config content
|
||||
monitorDir = config.GetString("comtrade_file_dir")
|
||||
}
|
||||
|
||||
func dirWatch(monitorDir string, addChan chan string) {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
pool, err := ants.NewPoolWithFunc(waveRecordConfig.ParseConcurrentQuantity, comtrade.ParseFunc)
|
||||
if err != nil {
|
||||
logger.Error("create dir watcher failed:", zap.Error(err))
|
||||
}
|
||||
defer watcher.Close()
|
||||
|
||||
err = watcher.Add(monitorDir)
|
||||
if err != nil {
|
||||
logger.Error("add monitor dir failed:", zap.Error(err))
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-watcher.Events:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if event.Op&fsnotify.Create == fsnotify.Create {
|
||||
logger.Info("file add in watcher dir", zap.String("file_name", event.Name))
|
||||
addChan <- event.Name
|
||||
}
|
||||
|
||||
if event.Op&fsnotify.Remove == fsnotify.Remove {
|
||||
logger.Info("file remove in watcher dir", zap.String("file_name", event.Name))
|
||||
}
|
||||
case err, ok := <-watcher.Errors:
|
||||
if !ok {
|
||||
logger.Error("monitor watcher error channel closed", zap.Bool("channel_status", ok))
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error("monitor watcher error occurred", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseComtradeFile(addChan chan string, comtradeMap sync.Map) {
|
||||
for {
|
||||
addFilePath, ok := <-addChan
|
||||
if !ok {
|
||||
logger.Error("monitor add channel closed", zap.Bool("channel_status", ok))
|
||||
return
|
||||
}
|
||||
|
||||
lastElement := filepath.Base(addFilePath)
|
||||
fileName := strings.Split(lastElement, ".")[0]
|
||||
fileExtension := filepath.Ext(lastElement)
|
||||
logger.Info("add comtrade file info", zap.String("file_path", addFilePath),
|
||||
zap.String("file_name", fileName), zap.String("file_extension", fileExtension))
|
||||
|
||||
switch fileExtension {
|
||||
case constant.ConfigFileSuffix:
|
||||
configFilePath := addFilePath
|
||||
|
||||
dataFilePath, exist := comtradeMap.Load(addFilePath)
|
||||
if exist {
|
||||
if dataFilePath == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
go parseComtradeData(configFilePath, dataFilePath.(string))
|
||||
} else {
|
||||
comtradeMap.Store(configFilePath, "")
|
||||
}
|
||||
case constant.DataFileSuffix:
|
||||
configFileName := strings.Join([]string{fileName, constant.ConfigFileSuffix}, "")
|
||||
configFilePath := filepath.Join(monitorDir, configFileName)
|
||||
logger.Info("config path of comtrade file", zap.String("file_path", configFilePath))
|
||||
exist := util.FileExists(configFilePath)
|
||||
if exist {
|
||||
comtradeMap.Store(configFilePath, addFilePath)
|
||||
addChan <- configFilePath
|
||||
continue
|
||||
}
|
||||
addChan <- addFilePath
|
||||
default:
|
||||
logger.Warn("no support file style", zap.String("file_style", fileExtension))
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseComtradeData(configFilePath, dataFilePath string) {
|
||||
c, err := comtrade.ParseComtrade(configFilePath, dataFilePath)
|
||||
if err != nil {
|
||||
logger.Error("parse comtrade file failed", zap.Error(err))
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
fmt.Println("parse succcess")
|
||||
fmt.Println(c.Conf.Ft)
|
||||
// TODO 增加文件存入mongodb
|
||||
|
||||
for i := 0; i < 15; i++ {
|
||||
if err = pool.Invoke(i); err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
go util.DirWatch(waveRecordConfig.MonitorDir, addChannel)
|
||||
|
||||
comtrade.ParseComtradeFile(waveRecordConfig.MonitorDir, addChannel, &comtradeMap)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func DirWatch(monitorDir string, addChan chan string) {
|
||||
logger := zap.L()
|
||||
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
logger.Error("create dir watcher failed:", zap.Error(err))
|
||||
}
|
||||
defer watcher.Close()
|
||||
|
||||
err = watcher.Add(monitorDir)
|
||||
if err != nil {
|
||||
logger.Error("add monitor dir failed:", zap.Error(err))
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-watcher.Events:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if event.Op&fsnotify.Create == fsnotify.Create {
|
||||
logger.Info("file add in watcher dir", zap.String("file_name", event.Name))
|
||||
addChan <- event.Name
|
||||
}
|
||||
|
||||
if event.Op&fsnotify.Remove == fsnotify.Remove {
|
||||
logger.Info("file remove in watcher dir", zap.String("file_name", event.Name))
|
||||
}
|
||||
case err, ok := <-watcher.Errors:
|
||||
if !ok {
|
||||
logger.Error("monitor watcher error channel closed", zap.Bool("channel_status", ok))
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error("monitor watcher error occurred", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue