// Package comtrade define related functions for comtrade data processing package comtrade import ( "context" "errors" "fmt" "path/filepath" "strings" "sync" "time" "wave_record/config" "wave_record/constant" "wave_record/database" "wave_record/util" "github.com/panjf2000/ants/v2" "github.com/yonwoo9/go-comtrade" "go.uber.org/zap" ) // ParseFunc define ants concourrent exec func var ParseFunc = func(parseConfig interface{}) { logger := zap.L() comtradeStorageConfig, ok := parseConfig.(config.ComtradeDataStorageConfig) if !ok { // TODO 增加日志报错中输出文件名 logger.Error("conversion parsing comtrade parameter type failed") return } comtradeData, err := parseComtradeData(comtradeStorageConfig.ConfigFilePath, comtradeStorageConfig.DataFilePath) if err != nil { // TODO 增加日志报错中输出文件名 logger.Error("parsing comtrade data failed", zap.Error(err)) return } collection := database.MongoDBClient().Database(comtradeStorageConfig.DBName).Collection(comtradeData.Conf.StationName) ctx, cancel := context.WithTimeout(comtradeStorageConfig.Ctx, 5*time.Second) defer cancel() err = database. StorageComtradeIntoMongoDB(ctx, comtradeData, collection, logger) if err != nil { // TODO 增加处理失败的文件名 logger.Error("stroage comtrade data info mongoDB failed", zap.Error(err)) } comtradeStorageConfig.DelChan <- comtradeStorageConfig.ConfigFilePath comtradeStorageConfig.DelChan <- comtradeStorageConfig.DataFilePath } func parseComtradeData(configFilePath, dataFilePath string) (*comtrade.Comtrade, error) { logger := zap.L() comtrade, err := comtrade.ParseComtrade(configFilePath, dataFilePath) if err != nil { logger.Error("parse comtrade file failed", zap.Error(err)) return nil, err } return comtrade, nil } // ParseComtradeFile define comtrade file parse func func ParseComtradeFile(ctx context.Context, monitorDir string, dbName string, addChan chan string, delChan chan string, comtradeMap *sync.Map, pool *ants.PoolWithFunc) { logger := zap.L() for { addFilePath, ok := <-addChan if !ok { logger.Error("monitor add channel closed", zap.Bool("channel_status", ok)) return } err := processComtradeFile(ctx, addFilePath, monitorDir, dbName, comtradeMap, addChan, delChan, pool, logger) if errors.Is(err, nil) { // TODO 写log error报警 logger.Error("process comtrade file failed", zap.Error(err)) continue } } } func processComtradeFile(ctx context.Context, addFilePath string, monitorDir string, dbName string, comtradeMap *sync.Map, addChan chan string, delChan chan string, pool *ants.PoolWithFunc, logger *zap.Logger) error { lastElement := filepath.Base(addFilePath) fileName := strings.Split(lastElement, ".")[0] fileExtension := filepath.Ext(lastElement) logger.Info("add comtrade file info", zap.String("file_path", addFilePath), zap.String("file_name", fileName), zap.String("file_extension", fileExtension)) switch fileExtension { case constant.ConfigFileSuffix: err := processConfigFile(ctx, dbName, addFilePath, delChan, comtradeMap, pool) if err != nil { // TODO 对error进行嵌套返回 return fmt.Errorf("%w", err) } case constant.DataFileSuffix: err := processDataFile(monitorDir, fileName, addFilePath, comtradeMap, addChan, logger) if err != nil { // TODO 对error进行嵌套返回 return fmt.Errorf("%w", err) } default: logger.Warn("no support file style", zap.String("file_style", fileExtension)) time.Sleep(5 * time.Second) } return nil } func processConfigFile(ctx context.Context, dbName string, configFilePath string, delchan chan string, comtradeMap *sync.Map, pool *ants.PoolWithFunc) error { dataFilePath, exist := comtradeMap.Load(configFilePath) if exist { if dataFilePath == "" { return errors.New("can not find dat file in map") } pool.Invoke(config.ComtradeDataStorageConfig{ Ctx: ctx, DelChan: delchan, DBName: dbName, ConfigFilePath: configFilePath, DataFilePath: dataFilePath.(string), }) return nil } comtradeMap.Store(configFilePath, "") return nil } func processDataFile(monitorDir string, fileName string, dataFilePath string, comtradeMap *sync.Map, addChan chan string, logger *zap.Logger) error { configFileName := strings.Join([]string{fileName, constant.ConfigFileSuffix}, "") configFilePath := filepath.Join(monitorDir, configFileName) logger.Info("config path of comtrade file", zap.String("file_path", configFilePath)) exist := util.FileExists(configFilePath) if exist { comtradeMap.Store(configFilePath, dataFilePath) addChan <- configFilePath return nil } addChan <- dataFilePath return nil } // MoveComtradeFile define comtrade file remove from comtradeMap and move comtrade file from monitor dir to backup dir func func MoveComtradeFile(backupDir string, comtradeMap *sync.Map, delChan chan string) { logger := zap.L() for { filePath := <-delChan fileName := filepath.Base(filePath) fileExtension := filepath.Ext(fileName) if fileExtension == constant.ConfigFileSuffix { comtradeMap.Delete(filePath) } backupFilePath := filepath.Join(backupDir, fileName) err := util.RemoveFile(filePath, backupFilePath) if err != nil { logger.Error("remove file to backup dir failed", zap.Error(err)) } } }