diff --git a/wave_record/comtrade/parse.go b/wave_record/comtrade/parse.go new file mode 100644 index 0000000..eeb3518 --- /dev/null +++ b/wave_record/comtrade/parse.go @@ -0,0 +1,86 @@ +package comtrade + +import ( + "path/filepath" + "strings" + "sync" + "time" + + "wave_record/config" + "wave_record/constant" + "wave_record/util" + + "github.com/yonwoo9/go-comtrade" + "go.uber.org/zap" +) + +var ParseFunc = func(parseConfig interface{}) { + logger := zap.L() + comtradeConfig, ok := parseConfig.(config.ComtradeFileConfig) + if !ok { + // TODO 增加日志报错中输出文件名 + logger.Error("conversion parsing comtrade parameter type failed") + return + } + + parseComtradeData(comtradeConfig.ConfigFilePath, *&comtradeConfig.DatafilePath) +} + +func ParseComtradeFile(monitorDir string, addChan chan string, comtradeMap *sync.Map) { + logger := zap.L() + + for { + addFilePath, ok := <-addChan + if !ok { + logger.Error("monitor add channel closed", zap.Bool("channel_status", ok)) + return + } + + lastElement := filepath.Base(addFilePath) + fileName := strings.Split(lastElement, ".")[0] + fileExtension := filepath.Ext(lastElement) + logger.Info("add comtrade file info", zap.String("file_path", addFilePath), + zap.String("file_name", fileName), zap.String("file_extension", fileExtension)) + + switch fileExtension { + case constant.ConfigFileSuffix: + configFilePath := addFilePath + + dataFilePath, exist := comtradeMap.Load(addFilePath) + if exist { + if dataFilePath == "" { + continue + } + + go parseComtradeData(configFilePath, dataFilePath.(string)) + } else { + comtradeMap.Store(configFilePath, "") + } + case constant.DataFileSuffix: + configFileName := strings.Join([]string{fileName, constant.ConfigFileSuffix}, "") + configFilePath := filepath.Join(monitorDir, configFileName) + logger.Info("config path of comtrade file", zap.String("file_path", configFilePath)) + exist := util.FileExists(configFilePath) + if exist { + comtradeMap.Store(configFilePath, addFilePath) + addChan <- configFilePath + continue + } + addChan <- addFilePath + default: + logger.Warn("no support file style", zap.String("file_style", fileExtension)) + time.Sleep(5 * time.Second) + } + } +} + +func parseComtradeData(configFilePath, dataFilePath string) (*comtrade.Comtrade, error) { + logger := zap.L() + + comtrade, err := comtrade.ParseComtrade(configFilePath, dataFilePath) + if err != nil { + logger.Error("parse comtrade file failed", zap.Error(err)) + return nil, err + } + return comtrade, nil +} diff --git a/wave_record/config/config.go b/wave_record/config/config.go new file mode 100644 index 0000000..34022c1 --- /dev/null +++ b/wave_record/config/config.go @@ -0,0 +1,56 @@ +// Package config define config struct of wave record project +package config + +import ( + "strings" + + "wave_record/log" + + "github.com/spf13/viper" +) + +// WaveRecordConfig is designed for wave record config struct +type WaveRecordConfig struct { + MonitorDir string + ParseConcurrentQuantity int // parse comtrade file concurrent quantity + MongoDBURI string + MongoDBDataBase string + LCfg log.LogConfig // log config +} + +// ComtradeFileConfig define config struct of parse comtrade data +type ComtradeFileConfig struct { + ConfigFilePath string + DatafilePath string +} + +// ReadAndInitConfig return wave record project config struct +func ReadAndInitConfig(configDir, configName, configType string) (waveRecordConfig WaveRecordConfig) { + config := viper.New() + config.AddConfigPath(configDir) + config.SetConfigName(configName) + config.SetConfigType(configType) + if err := config.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + panic("can not find conifg file") + } + panic(err) + } + + waveRecordConfig.MonitorDir = config.GetString("comtrade_monitor_dir") + // init mongodb config from config.yaml + mongoDBHost := config.GetString("mongodb_host") + mongoDBPort := config.GetString("mongodb_port") + waveRecordConfig.MongoDBURI = strings.Join([]string{"mongodb://", mongoDBHost, ":", mongoDBPort}, "") + waveRecordConfig.MongoDBDataBase = config.GetString("mongodb_database") + // init zap log config from config.yaml + waveRecordConfig.LCfg.Level = config.GetString("log_level") + waveRecordConfig.LCfg.FileName = config.GetString("log_filename") + waveRecordConfig.LCfg.MaxSize = config.GetInt("log_maxsize") + waveRecordConfig.LCfg.MaxBackups = config.GetInt("log_maxbackups") + waveRecordConfig.LCfg.MaxAge = config.GetInt("log_maxage") + + waveRecordConfig.ParseConcurrentQuantity = config.GetInt("parse_concurrent_quantity") + + return waveRecordConfig +} diff --git a/wave_record/config/config.yaml b/wave_record/config/config.yaml index 01b7408..2edc1c7 100644 --- a/wave_record/config/config.yaml +++ b/wave_record/config/config.yaml @@ -1 +1,13 @@ -comtrade_file_dir: /tmp +comtrade_monitor_dir: "/tmp" + +mongodb_host: "localhost" +mongodb_port: "27017" +mongodb_database: "wave_record" + +log_level: "debug" +log_filename: "/log/wave_record.log" +log_maxsize: 1 +log_maxbackups: 5 +log_maxage: 30 + +parse_concurrent_quantity: 10 diff --git a/wave_record/database/init.go b/wave_record/database/init.go new file mode 100644 index 0000000..c25a92c --- /dev/null +++ b/wave_record/database/init.go @@ -0,0 +1,34 @@ +package database + +import ( + "context" + "sync" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + client *mongo.Client + once sync.Once +) + +// GetMongoDBInstance return instance of mongoDB client +func GetMongoDBInstance(ctx context.Context, mongoDBURI string) *mongo.Client { + once.Do(func() { + client = initMongoDBClient(ctx, mongoDBURI) + }) + return client +} + +// initMongoDBClient return successfully initialized mongoDB client +func initMongoDBClient(ctx context.Context, mongoDBURI string) *mongo.Client { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + mongoDBClient, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoDBURI)) + if err != nil { + panic(err) + } + return mongoDBClient +} diff --git a/wave_record/database/mongodb_store.go b/wave_record/database/mongodb_store.go new file mode 100644 index 0000000..1b342c1 --- /dev/null +++ b/wave_record/database/mongodb_store.go @@ -0,0 +1,31 @@ +package database + +import ( + "context" + "time" + + "github.com/yonwoo9/go-comtrade" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.uber.org/zap" +) + +// StorageComtradeIntoMongoDB return the result of storing comtrade data into mongoDB +func StorageComtradeIntoMongoDB(ctx context.Context, comtradeData *comtrade.Comtrade, dbName string, client *mongo.Client, logger *zap.Logger) error { + collection := client.Database(dbName).Collection(comtradeData.Conf.StationName) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + comtradeBson, err := bson.Marshal(comtradeData) + if err != nil { + logger.Error("bson marshal comtrade data failed", zap.Error(err)) + return err + } + + _, err = collection.InsertOne(ctx, comtradeBson) + if err != nil { + logger.Error("insert comtrade data into mongoDB failed", zap.Error(err)) + return err + } + return nil +} diff --git a/wave_record/docker-compose.yaml b/wave_record/docker-compose.yaml new file mode 100644 index 0000000..e69de29 diff --git a/wave_record/go.mod b/wave_record/go.mod index e119b5d..f24ec8b 100644 --- a/wave_record/go.mod +++ b/wave_record/go.mod @@ -4,15 +4,22 @@ go 1.22.5 require ( github.com/fsnotify/fsnotify v1.7.0 + github.com/natefinch/lumberjack v2.0.0+incompatible + github.com/panjf2000/ants/v2 v2.10.0 github.com/spf13/viper v1.19.0 github.com/yonwoo9/go-comtrade v0.0.3 + go.mongodb.org/mongo-driver v1.16.0 go.uber.org/zap v1.21.0 ) require ( + github.com/BurntSushi/toml v1.4.0 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -21,11 +28,18 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.10.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/wave_record/go.sum b/wave_record/go.sum index e64da9a..71186aa 100644 --- a/wave_record/go.sum +++ b/wave_record/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -8,10 +10,14 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -23,6 +29,12 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= +github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= +github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -55,14 +67,26 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/yonwoo9/go-comtrade v0.0.3 h1:+Qbjn26MMXNacsGYdbYh4jNfMWiIdF8vRs6a0q/0xrU= github.com/yonwoo9/go-comtrade v0.0.3/go.mod h1:FhyIfNGvvEfVXkCzzc5Xsp8HBtHK9AQxF0d2jkp5d4g= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= +go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -75,32 +99,49 @@ go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -110,6 +151,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/wave_record/log/init.go b/wave_record/log/init.go new file mode 100644 index 0000000..6c5ea45 --- /dev/null +++ b/wave_record/log/init.go @@ -0,0 +1,77 @@ +// Package log define log struct of wave record project +package log + +import ( + "os" + "sync" + + "github.com/natefinch/lumberjack" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var ( + logger *zap.Logger + once sync.Once +) + +// LogConfig define log config of wave record project +type LogConfig struct { + Level string `json:"level"` // Level 最低日志等级，DEBUG收集INFO等级以上的日志 + FileName string `json:"file_name"` // FileName 日志文件位置 + MaxSize int `json:"max_size"` // MaxSize 进行切割之前,日志文件的最大大小(MB为单位)默认为100MB + MaxAge int `json:"max_age"` // MaxAge 是根据文件名中编码的时间戳保留旧日志文件的最大天数。 + MaxBackups int `json:"max_backups"` // MaxBackups 是要保留的旧日志文件的最大数量。默认是保留所有旧的日志文件(尽管 MaxAge 可能仍会导致它们被删除) +} + +// getEncoder responsible for setting the log format for encoding +func getEncoder() zapcore.Encoder { + encodeConfig := zap.NewProductionEncoderConfig() + // serialization time eg:2006-01-02 15:04:05 + encodeConfig.EncodeTime = zapcore.TimeEncoderOfLayout("2006-01-02 15:04:05") + encodeConfig.TimeKey = "time" + encodeConfig.EncodeLevel = zapcore.CapitalLevelEncoder + encodeConfig.EncodeCaller = zapcore.ShortCallerEncoder + return zapcore.NewJSONEncoder(encodeConfig) +} + +// getLogWriter responsible for setting the location of log storage +func getLogWriter(filename string, maxsize, maxBackup, maxAge int) zapcore.WriteSyncer { + lumberJackLogger := &lumberjack.Logger{ + Filename: filename, // log file position + MaxSize: maxsize, // log file maxsize + MaxAge: maxAge, // maximum number of day files retained + MaxBackups: maxBackup, // maximum number of old files retained + Compress: false, // whether to compress + } + syncFile := zapcore.AddSync(lumberJackLogger) + // TODO:增加调试输出到控制台设置，其他模式无控制台数据 + syncConsole := zapcore.AddSync(os.Stderr) + return zapcore.NewMultiWriteSyncer(syncFile, syncConsole) +} + +// initLogger return successfully initialized zap logger +func initLogger(lCfg LogConfig) *zap.Logger { + writeSyncer := getLogWriter(lCfg.FileName, lCfg.MaxSize, lCfg.MaxBackups, lCfg.MaxAge) + encoder := getEncoder() + + l := new(zapcore.Level) + err := l.UnmarshalText([]byte(lCfg.Level)) + if err != nil { + panic(err) + } + + core := zapcore.NewCore(encoder, writeSyncer, l) + logger = zap.New(core, zap.AddCaller()) + zap.ReplaceGlobals(logger) + + return logger +} + +// GetLoggerInstance return instance of zap logger +func GetLoggerInstance(lCfg LogConfig) *zap.Logger { + once.Do(func() { + logger = initLogger(lCfg) + }) + return logger +} diff --git a/wave_record/main.go b/wave_record/main.go index c955602..64251f1 100644 --- a/wave_record/main.go +++ b/wave_record/main.go @@ -1,20 +1,21 @@ package main import ( + "context" "flag" "fmt" - "path/filepath" - "strings" "sync" - "time" - "wave_record/constant" + "wave_record/comtrade" + "wave_record/config" + "wave_record/database" + "wave_record/log" "wave_record/util" - "github.com/fsnotify/fsnotify" - "github.com/spf13/viper" - "github.com/yonwoo9/go-comtrade" + "go.mongodb.org/mongo-driver/mongo" "go.uber.org/zap" + + "github.com/panjf2000/ants/v2" ) var ( @@ -24,133 +25,52 @@ var ( ) var ( - monitorDir string - addChannel chan string - comtradeMap sync.Map - logger *zap.Logger + waveRecordConfig config.WaveRecordConfig + addChannel chan string + comtradeMap sync.Map + mongoDBClient *mongo.Client + logger *zap.Logger ) func init() { addChannel = make(chan string, 10) - logger, _ = zap.NewProduction() } func main() { flag.Parse() + ctx := context.TODO() + + waveRecordConfig = config.ReadAndInitConfig(*comtradeConfigDir, *comtradeConfigName, *comtradeConfigType) + + // init mongoDBClient + mongoDBClient = database.GetMongoDBInstance(ctx, waveRecordConfig.MongoDBURI) + + // init logger + logger = log.GetLoggerInstance(waveRecordConfig.LCfg) + + defer func() { + if err := mongoDBClient.Disconnect(ctx); err != nil { + panic(err) + } + }() + defer logger.Sync() - readConfig() - logger.Info("comtrade monitor Dir", zap.String("monitor_dir", monitorDir)) - go dirWatch(monitorDir, addChannel) - parseComtradeFile(addChannel, comtradeMap) -} + defer ants.Release() -func readConfig() { - config := viper.New() - config.AddConfigPath(*comtradeConfigDir) - config.SetConfigName(*comtradeConfigName) - config.SetConfigType(*comtradeConfigType) - if err := config.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - panic("can not find conifg file") - } - panic(err) - } - // get config content - monitorDir = config.GetString("comtrade_file_dir") -} - -func dirWatch(monitorDir string, addChan chan string) { - watcher, err := fsnotify.NewWatcher() + pool, err := ants.NewPoolWithFunc(waveRecordConfig.ParseConcurrentQuantity, comtrade.ParseFunc) if err != nil { - logger.Error("create dir watcher failed:", zap.Error(err)) - } - defer watcher.Close() - - err = watcher.Add(monitorDir) - if err != nil { - logger.Error("add monitor dir failed:", zap.Error(err)) - } - - for { - select { - case event, ok := <-watcher.Events: - if !ok { - return - } - - if event.Op&fsnotify.Create == fsnotify.Create { - logger.Info("file add in watcher dir", zap.String("file_name", event.Name)) - addChan <- event.Name - } - - if event.Op&fsnotify.Remove == fsnotify.Remove { - logger.Info("file remove in watcher dir", zap.String("file_name", event.Name)) - } - case err, ok := <-watcher.Errors: - if !ok { - logger.Error("monitor watcher error channel closed", zap.Bool("channel_status", ok)) - return - } - - logger.Error("monitor watcher error occurred", zap.Error(err)) - } - } -} - -func parseComtradeFile(addChan chan string, comtradeMap sync.Map) { - for { - addFilePath, ok := <-addChan - if !ok { - logger.Error("monitor add channel closed", zap.Bool("channel_status", ok)) - return - } - - lastElement := filepath.Base(addFilePath) - fileName := strings.Split(lastElement, ".")[0] - fileExtension := filepath.Ext(lastElement) - logger.Info("add comtrade file info", zap.String("file_path", addFilePath), - zap.String("file_name", fileName), zap.String("file_extension", fileExtension)) - - switch fileExtension { - case constant.ConfigFileSuffix: - configFilePath := addFilePath - - dataFilePath, exist := comtradeMap.Load(addFilePath) - if exist { - if dataFilePath == "" { - continue - } - - go parseComtradeData(configFilePath, dataFilePath.(string)) - } else { - comtradeMap.Store(configFilePath, "") - } - case constant.DataFileSuffix: - configFileName := strings.Join([]string{fileName, constant.ConfigFileSuffix}, "") - configFilePath := filepath.Join(monitorDir, configFileName) - logger.Info("config path of comtrade file", zap.String("file_path", configFilePath)) - exist := util.FileExists(configFilePath) - if exist { - comtradeMap.Store(configFilePath, addFilePath) - addChan <- configFilePath - continue - } - addChan <- addFilePath - default: - logger.Warn("no support file style", zap.String("file_style", fileExtension)) - time.Sleep(5 * time.Second) - } - } -} - -func parseComtradeData(configFilePath, dataFilePath string) { - c, err := comtrade.ParseComtrade(configFilePath, dataFilePath) - if err != nil { - logger.Error("parse comtrade file failed", zap.Error(err)) + fmt.Println(err) return } - fmt.Println("parse succcess") - fmt.Println(c.Conf.Ft) - // TODO 增加文件存入mongodb + + for i := 0; i < 15; i++ { + if err = pool.Invoke(i); err != nil { + fmt.Println(err) + } + } + + go util.DirWatch(waveRecordConfig.MonitorDir, addChannel) + + comtrade.ParseComtradeFile(waveRecordConfig.MonitorDir, addChannel, &comtradeMap) } diff --git a/wave_record/util/dir_monitor.go b/wave_record/util/dir_monitor.go new file mode 100644 index 0000000..2b68324 --- /dev/null +++ b/wave_record/util/dir_monitor.go @@ -0,0 +1,46 @@ +package util + +import ( + "github.com/fsnotify/fsnotify" + "go.uber.org/zap" +) + +func DirWatch(monitorDir string, addChan chan string) { + logger := zap.L() + + watcher, err := fsnotify.NewWatcher() + if err != nil { + logger.Error("create dir watcher failed:", zap.Error(err)) + } + defer watcher.Close() + + err = watcher.Add(monitorDir) + if err != nil { + logger.Error("add monitor dir failed:", zap.Error(err)) + } + + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + + if event.Op&fsnotify.Create == fsnotify.Create { + logger.Info("file add in watcher dir", zap.String("file_name", event.Name)) + addChan <- event.Name + } + + if event.Op&fsnotify.Remove == fsnotify.Remove { + logger.Info("file remove in watcher dir", zap.String("file_name", event.Name)) + } + case err, ok := <-watcher.Errors: + if !ok { + logger.Error("monitor watcher error channel closed", zap.Bool("channel_status", ok)) + return + } + + logger.Error("monitor watcher error occurred", zap.Error(err)) + } + } +}