diff --git a/.idea/misc.xml b/.idea/misc.xml
index c257415..cc856b5 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -3,4 +3,5 @@
+
\ No newline at end of file
diff --git a/FC_ML_Baseline/FC_ML_Baseline_Data_Handler/Data_Load.py b/FC_ML_Baseline/FC_ML_Baseline_Data_Handler/Data_Load.py
index bfda1f3..b826b08 100644
--- a/FC_ML_Baseline/FC_ML_Baseline_Data_Handler/Data_Load.py
+++ b/FC_ML_Baseline/FC_ML_Baseline_Data_Handler/Data_Load.py
@@ -55,7 +55,7 @@ from FC_ML_Tool.Serialization import parse_json_file
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='导入数据文件参数')
- parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\param.json',
+ parser.add_argument('--param', default='D:\liyong\project\ModelTrainingPython\FC_ML_Baseline\FC_ML_Baseline_Test\param.json',
help='配置参数文件绝对路径')
parser.add_argument('--export', default='source.json',
help='导出JSON文件名')
diff --git a/FC_ML_Baseline/FC_ML_Baseline_Predict/Model_Pred.py b/FC_ML_Baseline/FC_ML_Baseline_Predict/Model_Pred.py
index 8dea61e..e8c89f1 100644
--- a/FC_ML_Baseline/FC_ML_Baseline_Predict/Model_Pred.py
+++ b/FC_ML_Baseline/FC_ML_Baseline_Predict/Model_Pred.py
@@ -2,7 +2,6 @@ import argparse
import json
import torch
-from openpyxl.styles.builtins import output
from FC_ML_Data.FC_ML_Data_Process.Data_Process_Normalization import Normalizer
from FC_ML_NN_Model.Poly_Model import PolyModel
@@ -10,18 +9,19 @@ from FC_ML_Tool.Serialization import parse_json_file
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='代理模型训练参数输入')
- parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
+ parser.add_argument('--param', default='D:\liyong\project\ModelTrainingPython\FC_ML_Baseline\FC_ML_Baseline_Test\pred\param.json',
help='配置参数文件绝对路径')
args = parser.parse_args()
params = parse_json_file(args.param)
+ print(params)
source_dir = params["path"] + "/"
model_file = source_dir + params["modelFile"]
inputs = []
- names = []
+ names = params["output"]["names"]
#获取输入特征
for input_value in params["input"]:
inputs.append(input_value["value"])
- names.append(input_value["name"])
+ # names.append(input_value["name"])
#记载模型进行预测
input_size = params["modelParams"]["inputSize"]
output_size = params["modelParams"]["outputSize"]
@@ -36,21 +36,23 @@ if __name__ == "__main__":
normalization_max = params["modelParams"]["normalizerMax"]
normalization_min = params["modelParams"]["normalizerMin"]
normalizer = Normalizer(method=normalization_type)
- normalizer.load_params(normalization_type,normalization_min,normalization_max)
+ normalizer.load_params(normalization_type,normalization_min[0:input_size],normalization_max[0:input_size])
input_data = normalizer.transform(torch.tensor(inputs))
#执行模型预测
with torch.no_grad():
output_data = model(input_data)
- print(f"Prediction result: {output_data.item():.4f}")
+ # print(f"Prediction result: {output_data.item().tolist():.4f}")
+ normalizer.load_params(normalization_type, normalization_min[-output_size:], normalization_max[-output_size:])
output_data_ori = normalizer.inverse_transform(output_data)
- print(f"Prediction real result: {output_data_ori.item():.4f}")
+ # print(f"Prediction real result: {output_data_ori.item().tolist():.4f}")
#输出预测结果到文件中
output_datas = output_data_ori.tolist()
json_str = {}
+
if len(output_datas) == len(names):
for i in range(len(names)):
json_str[names[i]] = output_datas[i]
- with open(source_dir + "forecast.json", ) as f:
+ with open(source_dir + "forecast.json","w") as f:
f.write(json.dumps(json_str, indent=None, ensure_ascii=False))
diff --git a/FC_ML_Baseline/FC_ML_Baseline_Test/Data_Handle/param.json b/FC_ML_Baseline/FC_ML_Baseline_Test/Data_Handle/param.json
index 9256b84..090f78f 100644
--- a/FC_ML_Baseline/FC_ML_Baseline_Test/Data_Handle/param.json
+++ b/FC_ML_Baseline/FC_ML_Baseline_Test/Data_Handle/param.json
@@ -1,4 +1,4 @@
{
"files": ["sample1.CSV"],
- "path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle"
+ "path": "D:\\liyong\\project\\ModelTrainingPython\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle"
}
diff --git a/FC_ML_Baseline/FC_ML_Baseline_Test/Train/param.json b/FC_ML_Baseline/FC_ML_Baseline_Test/Train/param.json
index 853f4ae..23d3e62 100644
--- a/FC_ML_Baseline/FC_ML_Baseline_Test/Train/param.json
+++ b/FC_ML_Baseline/FC_ML_Baseline_Test/Train/param.json
@@ -1,20 +1,23 @@
{
- "path": ["D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Train"],
+ "files": ["sample1.CSV"],
+ "path": "D:\\liyong\\project\\ModelTrainingPython\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle",
"algorithmParam": {
+ "inputSize": 9,
+ "outputSize": 8,
"algorithm": "多项式拟合",
"activateFun": "sigmod",
"lossFun": "l1",
"optimizeFun": "sgd",
- "exportFormat": ".onnx",
+ "exportFormat": "bin",
"trainingRatio": 80,
"loadSize": 32,
"studyPercent": 0.001,
"stepCounts": 3,
- "roundPrint": 11,
- "round": 1001,
- "preDisposeData": false,
+ "roundPrint": 10,
+ "round": 300,
+ "preDisposeData": true,
"disposeMethod": "minmax",
- "dataNoOrder": false
+ "dataNoOrder": true
},
"algorithm": "基础神经网络NN"
}
\ No newline at end of file
diff --git a/FC_ML_Baseline/FC_ML_Baseline_Test/pred/param.json b/FC_ML_Baseline/FC_ML_Baseline_Test/pred/param.json
index d5cd2ca..bef4cf6 100644
--- a/FC_ML_Baseline/FC_ML_Baseline_Test/pred/param.json
+++ b/FC_ML_Baseline/FC_ML_Baseline_Test/pred/param.json
@@ -1,20 +1,51 @@
{
- "modelFile": "model.onnx",
- "path": "D:\\liyong\\project\\TVS_ML\\FC_ML_Baseline\\FC_ML_Baseline_Test\\pred",
+ "modelFile": "model.bin",
+ "path": "D:\\liyong\\project\\ModelTrainingPython\\FC_ML_Baseline\\FC_ML_Baseline_Test\\Data_Handle",
"modelParams": {
- "inputSize": 3,
- "outputSize": 3,
+ "inputSize": 9,
+ "outputSize": 8,
"normalizerType": "minmax",
- "normalizerMax": 100,
- "normalizerMin": 10
+ "normalizerMax": [2000.0,575.9771118164062,5999.64208984375,5806.2333984375,6711.77880859375,99.99962615966797,99.99884796142578,-29.81661605834961,59.998504638671875,27.299999237060547,5.230000019073486,131.0,8.170000076293945,11.899999618530273,-0.8949999809265137,27.100000381469727,17.899999618530273],
+ "normalizerMin": [0.10000000149011612,5.022274971008301,2.0320935249328613,0.21287846565246582,3853.6533203125,0.019815441220998764,0.0033870770130306482,-29.81661605834961,0.0007396229775622487,-37.900001525878906,0.06520000100135803,-9.699999809265137,2.0299999713897705,-32.900001525878906,-32.70000076293945,-29.0,-29.600000381469727]
},
"input": [
{
- "name": "质量",
- "value": 1
+ "name": "param1",
+ "value": 0.1
}, {
- "name": "系数",
- "value": 2
+ "name": "param1",
+ "value": 371.6669936
+ },
+ {
+ "name": "param1",
+ "value": 3483.012088
+ },
+ {
+ "name": "param1",
+ "value": 4333.292092
+ },
+ {
+ "name": "param1",
+ "value": 5582.788747
+ },
+ {
+ "name": "param1",
+ "value": 22.33362393
+ },
+ {
+ "name": "param1",
+ "value": 74.76711286
+ },
+ {
+ "name": "param1",
+ "value": -29.816617
+ },
+ {
+ "name": "param1",
+ "value": 17.14707502
}
- ]
+ ],
+ "output": {
+ "names": ["label1","label2","label3","label4","label5","label6","label7","label8"]
+ }
}
\ No newline at end of file
diff --git a/FC_ML_Baseline/FC_ML_Baseline_Train/Train_Proxy_Model.py b/FC_ML_Baseline/FC_ML_Baseline_Train/Train_Proxy_Model.py
index 436182e..9476b52 100644
--- a/FC_ML_Baseline/FC_ML_Baseline_Train/Train_Proxy_Model.py
+++ b/FC_ML_Baseline/FC_ML_Baseline_Train/Train_Proxy_Model.py
@@ -35,26 +35,25 @@
*/
'''
import argparse
-from locale import normalize
+from copy import deepcopy
import torch
from torch.utils.data import TensorDataset, DataLoader
-from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_data_from_csv_feature, get_train_data_from_csv
+from FC_ML_Data.FC_ML_Data_Load.Data_Load_Excel import get_train_data_from_csv
from FC_ML_Data.FC_ML_Data_Output.Data_Output_Pytorch import export_model
from FC_ML_Loss_Function.Loss_Function_Selector import LossFunctionSelector
-from FC_ML_Model.Model_Train_Data import TrainData
from FC_ML_NN_Model.Poly_Model import PolyModel
from FC_ML_Optim_Function.Optimizer_Selector import OptimizerSelector
from FC_ML_Tool.Serialization import parse_json_file
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='代理模型训练参数输入')
- parser.add_argument('--param', default='D:\liyong\project\TVS_ML\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
+ parser.add_argument('--param', default='D:\liyong\project\ModelTrainingPython\FC_ML_Baseline\FC_ML_Baseline_Test\Train\param.json',
help='配置参数文件绝对路径')
args = parser.parse_args()
params = parse_json_file(args.param)
- train_data = torch.tensor()
+ print(params)
# print(params)
#获取训练参数
input_Size = params["algorithmParam"]["inputSize"]#输入特征维度
@@ -73,11 +72,15 @@ if __name__ == '__main__':
dispose_method = params["algorithmParam"]["disposeMethod"] # 数据预处理方法
data_no_order = params["algorithmParam"]["dataNoOrder"] # 训练数据是否乱序处理
#加载所有训练数据
+ train_data = []
source_dir = params["path"] + "/"
for data_file in params["files"]:
data_file_path = source_dir + data_file
ori_data,normalize = get_train_data_from_csv(data_file_path,pre_dispose_data,dispose_method)
- torch.cat((train_data,ori_data),dim=0)#按行拼接
+ if len(train_data) == 0:
+ train_data = deepcopy(ori_data)
+ else:
+ train_data = torch.cat((train_data,ori_data),dim=0)#按行拼接
#拆分测试集和训练集
split = int(training_ratio / 100 * len(train_data))
train_dataset = TensorDataset(train_data[:split,0:input_Size], train_data[:split,input_Size:])
@@ -140,11 +143,11 @@ if __name__ == '__main__':
#每100次迭代输出一次损失数值
if epoch % round_print == 0:
print(
- f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1")
+ f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | Loss Factor: {avg_train_loss / avg_test_loss:.2f}:1")
with open(source_dir + "training.log", "a") as f:
- f.write(f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | 损失比: {avg_train_loss / avg_test_loss:.2f}:1\n") # 自动换行追加
+ f.write(f"Epoch {epoch} | Train Loss: {avg_train_loss:.4f} | Test Loss: {avg_test_loss:.4f} | Loss Factor: {avg_train_loss / avg_test_loss:.2f}:1\n") # 自动换行追加
#导出训练后的模型
- export_model(model,source_dir,"model",export_format)
+ export_model(model,source_dir,"model",export_format,torch.randn(1, input_Size))
diff --git a/FC_ML_Data/FC_ML_Data_Load/Data_Load_Excel.py b/FC_ML_Data/FC_ML_Data_Load/Data_Load_Excel.py
index 36cc3e8..e68687c 100644
--- a/FC_ML_Data/FC_ML_Data_Load/Data_Load_Excel.py
+++ b/FC_ML_Data/FC_ML_Data_Load/Data_Load_Excel.py
@@ -182,7 +182,7 @@ def get_data_from_csv_feature(data_path,skip_rows = 100,sample_rows = 100,normal
sampled_indices = torch.arange(0, len(data_ori), skip_rows) # 记录行号
return label_name,source_data,normalizer.params["min"],normalizer.params["max"],normalizer.params["mean"],sampled_indices,data_sample
-def get_train_data_from_csv(data_path,normalization = false,normalization_type = 'minmax'):
+def get_train_data_from_csv(data_path,normalization = True,normalization_type = 'minmax'):
"""读取csv数据文件并生成标准化训练数据
Args:
data_path (str): 文件绝对路径
@@ -196,6 +196,8 @@ def get_train_data_from_csv(data_path,normalization = false,normalization_type =
Examples:
get_data_from_csv_feature("D://test.excel")
+ :param normalization_type:
+ :param normalization:
"""
# 读取前xx行数据
df = pd.read_csv(data_path,encoding='gbk')
diff --git a/FC_ML_Data/FC_ML_Data_Output/Data_Output_Pytorch.py b/FC_ML_Data/FC_ML_Data_Output/Data_Output_Pytorch.py
index fdf10f8..476d238 100644
--- a/FC_ML_Data/FC_ML_Data_Output/Data_Output_Pytorch.py
+++ b/FC_ML_Data/FC_ML_Data_Output/Data_Output_Pytorch.py
@@ -3,7 +3,7 @@ import torch
def export_model_pt(model,target,name = "model"):
script_model = torch.jit.script(model) # 或 torch.jit.trace(model, input)
- script_model.save(target + name + ".pt")
+ script_model.save(target + name + ".pth")
#2 通用格式导出
def export_model_onnx(model,input_tensor,target,name="model"):
torch.onnx.export(model, input_tensor, target+ name + ".onnx")
@@ -11,12 +11,12 @@ def export_model_onnx(model,input_tensor,target,name="model"):
def export_model_bin(model,target,name = "weights"):
torch.save(model.state_dict(), target + name + ".bin")
-def export_model(model,target,file_name,name):
+def export_model(model,target,file_name,name,input_tensor):
if name == 'bin':
return export_model_bin(model,target,file_name)
if name == 'onnx':
- return export_model_onnx(model,target,file_name)
- if name == 'pt':
- return export_model_bin(model,target,file_name)
+ return export_model_onnx(model,input_tensor,target,file_name)
+ if name == 'pth':
+ return export_model_pt(model,target,file_name)
else:
raise ValueError(f"不支持的导出类型")
\ No newline at end of file
diff --git a/FC_ML_Data/FC_ML_Data_Process/Data_Process_Normalization.py b/FC_ML_Data/FC_ML_Data_Process/Data_Process_Normalization.py
index 1bfd37c..6057dfe 100644
--- a/FC_ML_Data/FC_ML_Data_Process/Data_Process_Normalization.py
+++ b/FC_ML_Data/FC_ML_Data_Process/Data_Process_Normalization.py
@@ -20,10 +20,10 @@ class Normalizer:
self.params['max_abs'] = data.abs().max(dim=0)[0]
return self
- def load_params(self,method = "minmax",min_in = 0,max_in = 0,mean_in =0,std=0,max_abs=0):
+ def load_params(self,method = "minmax",min_in = [],max_in = [],mean_in =[],std=[],max_abs=[]):
self.method = method
- self.params['min'] = min_in
- self.params['max'] = max_in
+ self.params['min'] = torch.tensor(min_in)
+ self.params['max'] = torch.tensor(max_in)
self.params['mean'] = mean_in
self.params['std'] = std
self.params['max_abs'] = max_abs