新建仓库维护数据预测项目

FC_ML_Baseline/FC_ML_Baseline_Train/Train_Proxy_Model.py

@@ -0,0 +1,151 @@
+'''
+/**
+ * 开始训练脚本
+ * 传参：python xxx.py 源文件路径 文件名  /xx/xx/param.json
+ * param.json 包含：
+ *          {
+ *
+ *            path: '/xx/xx/xx',//源文件路径
+ *            algorithmParam:{
+                algorithm: '多项式拟合',
+                activateFun: 'sigmod',
+                lossFun: 'l1',
+                optimizeFun: 'sgd',
+                exportFormat: '.onnx',
+                trainingRatio: 80,
+                loadSize: 32,
+                studyPercent: 0.001,
+                stepCounts: 3,
+                roundPrint: 11,
+                round: 1001,
+                preDisposeData: false,
+                disposeMethod: 'minmax',
+                dataNoOrder: false,
+              },
+ *            algorithm: '基础神经网络NN',//线性拟合 多项式拟合 基础神经网络NN 循环神经网络RNN 长短记忆神经网络LSTM
+ *          }
+ * 生成training.json文件放到  源文件路径下
+ * training.json文件包含 ：
+ *          日志曲线：logCurveData
+ *          表格数据：sourceData
+ *          稳态必传：averageData
+ *          动态必传：sourceCurveData
+ *
+ * 生成 training.log日志文件放到  源文件路径下
+ */
+'''
+import argparse
+from locale import normalize
+
+import torch
+from torch.utils.data import TensorDataset, DataLoader
+
+from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_data_from_csv_feature, get_train_data_from_csv
+from FC_ML_Data.FC_ML_Data_Output.Data_Output_Pytorch import export_model
+from FC_ML_Loss_Function.Loss_Function_Selector import LossFunctionSelector
+from FC_ML_Model.Model_Train_Data import TrainData
+from FC_ML_NN_Model.Poly_Model import PolyModel
+from FC_ML_Optim_Function.Optimizer_Selector import OptimizerSelector
+from FC_ML_Tool.Serialization import parse_json_file
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='代理模型训练参数输入')
+    parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
+                        help='配置参数文件绝对路径')
+    args = parser.parse_args()
+    params = parse_json_file(args.param)
+    train_data = torch.tensor()
+    # print(params)
+    #获取训练参数
+    input_Size = params["algorithmParam"]["inputSize"]#输入特征维度
+    output_Size = params["algorithmParam"]["outputSize"]#输出标签维度
+    algorithm = params["algorithmParam"]["algorithm"]#神经网络模型
+    activate_fun = params["algorithmParam"]["activateFun"]#激活函数类型
+    loss_fun = params["algorithmParam"]["lossFun"]#损失函数
+    optimize_fun = params["algorithmParam"]["optimizeFun"]#优化函数
+    export_format = params["algorithmParam"]["exportFormat"]#导出训练模型格式
+    training_ratio = params["algorithmParam"]["trainingRatio"]#训练数据占比
+    load_size = params["algorithmParam"]["loadSize"]#同时最大加载训练数据宽度
+    study_percent = params["algorithmParam"]["studyPercent"]#学习率
+    round_print = params["algorithmParam"]["roundPrint"]#训练多少轮进行打印
+    round_count = params["algorithmParam"]["round"]#总训练轮次
+    pre_dispose_data = params["algorithmParam"]["preDisposeData"]#是否进行数据预处理
+    dispose_method = params["algorithmParam"]["disposeMethod"]  # 数据预处理方法
+    data_no_order = params["algorithmParam"]["dataNoOrder"]  # 训练数据是否乱序处理
+    #加载所有训练数据
+    source_dir = params["path"] + "/"
+    for data_file in params["files"]:
+        data_file_path = source_dir + data_file
+        ori_data,normalize = get_train_data_from_csv(data_file_path,pre_dispose_data,dispose_method)
+        torch.cat((train_data,ori_data),dim=0)#按行拼接
+    #拆分测试集和训练集
+    split = int(training_ratio / 100 * len(train_data))
+    train_dataset = TensorDataset(train_data[:split,0:input_Size], train_data[:split,input_Size:])
+    test_dataset = TensorDataset(train_data[split:,0:input_Size], train_data[split:,input_Size:])
+    print(train_dataset,test_dataset)
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size = load_size, #批量加载数据
+        shuffle = data_no_order #数据打乱
+    )
+
+    test_loader = DataLoader(
+        test_dataset,
+        batch_size= load_size, #批量加载数据
+        shuffle=False #验证集默认不打乱
+    )
+
+    #初始化模型
+    DEVICE = torch.device("cpu")#默认CPU训练
+    model = PolyModel(input_size = input_Size,output_size=output_Size).to(DEVICE)
+    #初始化损失函数
+    loss_selector = LossFunctionSelector()
+    criterion = loss_selector.get_loss(loss_fun)
+    #初始化优化器
+    optim_selector = OptimizerSelector()
+
+    optimizer = optim_selector.get_optimizer(model.parameters(),optimize_fun, lr=study_percent)
+    # 在训练循环前初始化损失记录列表
+    train_losses = []
+    test_losses = []
+    for epoch in range(round_count):
+        #模型启用训练模式
+        epoch_train_loss = 0
+        model.train()
+        for features,labels in train_loader:
+            features,labels= features.to(DEVICE),labels.to(DEVICE)
+            pred = model(features)
+            loss = criterion(pred.squeeze(), labels)
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            #计算损失
+            epoch_train_loss += loss.item() * features.size(0)
+
+        # 计算平均训练损失并记录
+        avg_train_loss = epoch_train_loss / len(train_loader.dataset)
+        train_losses.append(avg_train_loss)
+
+        #模型启用评估模式
+        model.eval()
+        test_loss = 0
+        with torch.no_grad():#关闭梯度下降
+            for features, labels in test_loader:
+                features, labels = features.to(DEVICE), labels.to(DEVICE)
+                preds = model(features)
+                test_loss += criterion(preds.squeeze(), labels).item() * features.size(0)
+
+        avg_test_loss = test_loss / len(test_loader.dataset)
+        test_losses.append(avg_test_loss)
+        #每100次迭代输出一次损失数值
+        if epoch % round_print == 0:
+            print(
+                f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1")
+            with open(source_dir + "training.log", "a") as f:
+                f.write(f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1\n")  # 自动换行追加
+    #导出训练后的模型
+    export_model(model,source_dir,"model",export_format)
+
+
+
+