AcidificationModel/api/model_optimize/data_increase.py

# 导入常用包
import os
import pandas as pd
import numpy as np
from PIL import Image
from model_saver import save_model

# 机器学习模型导入
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import GradientBoostingRegressor as GBSTR
from sklearn.neighbors import KNeighborsRegressor
from xgboost import XGBRegressor as XGBR

# 导入数据处理函数
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler

# 导入评分函数
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error 
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import accuracy_score
from sklearn.metrics import log_loss
from sklearn.metrics import roc_auc_score

import pickle
from pathlib import Path

# 定义数据集配置
DATASET_CONFIGS = {
    'soil_acid_9features': {
        'file_path': 'model_optimize/data/data_filt.xlsx',
        'x_columns': range(1, 10),  # 9个特征(包含delta_ph)
        'y_column': -1,  # 105_day_ph
        'feature_names': [
            'organic_matter',        # OM g/kg
            'chloride',              # CL g/kg
            'cec',                   # CEC cmol/kg
            'h_concentration',       # H+ cmol/kg
            'hn',                    # HN mg/kg
            'al_concentration',      # Al3+ cmol/kg
            'free_alumina',          # Free alumina g/kg
            'free_iron',             # Free iron oxides g/kg
            'delta_ph'               # ΔpH
        ],
        'target_name': 'target_ph'
    },
    'soil_acid_8features': {
        'file_path': 'model_optimize/data/data_filt - 副本.xlsx',
        'x_columns': range(1, 9),  # 8个特征
        'y_column': -2,  # delta_ph
        'feature_names': [
            'organic_matter',        # OM g/kg
            'chloride',              # CL g/kg
            'cec',                   # CEC cmol/kg
            'h_concentration',       # H+ cmol/kg
            'hn',                    # HN mg/kg
            'al_concentration',      # Al3+ cmol/kg
            'free_alumina',          # Free alumina g/kg
            'free_iron',             # Free iron oxides g/kg
        ],
        'target_name': 'target_ph'
    },
    'soil_acid_8features_original': {
        'file_path': 'model_optimize/data/data_filt.xlsx',
        'x_columns': range(1, 9),  # 8个特征
        'y_column': -2,  # delta_ph
        'feature_names': [
            'organic_matter',        # OM g/kg
            'chloride',              # CL g/kg
            'cec',                   # CEC cmol/kg
            'h_concentration',       # H+ cmol/kg
            'hn',                    # HN mg/kg
            'al_concentration',      # Al3+ cmol/kg
            'free_alumina',          # Free alumina g/kg
            'free_iron',             # Free iron oxides g/kg
        ],
        'target_name': 'target_ph'
    },
    'soil_acid_6features': {
        'file_path': 'model_optimize/data/data_reflux2.xlsx',
        'x_columns': range(0, 6),  # 6个特征
        'y_column': -1,  # delta_ph
        'feature_names': [
            'organic_matter',        # OM g/kg
            'chloride',              # CL g/kg
            'cec',                   # CEC cmol/kg
            'h_concentration',       # H+ cmol/kg
            'hn',                    # HN mg/kg
            'al_concentration',      # Al3+ cmol/kg
        ],
        'target_name': 'delta_ph'
    },
    'acidity_reduce': {
        'file_path': 'model_optimize/data/Acidity_reduce.xlsx',
        'x_columns': range(1, 6),  # 5个特征
        'y_column': 0,  # 1/b
        'feature_names': [
            'pH',        
            'OM',              
            'CL', 
            'H',
            'Al'
        ],
        'target_name': 'target'
    },
    'acidity_reduce_new': {
        'file_path': 'model_optimize/data/Acidity_reduce_new.xlsx',
        'x_columns': range(1, 6),  # 5个特征
        'y_column': 0,  # 1/b
        'feature_names': [
            'pH',        
            'OM',              
            'CL', 
            'H',
            'Al'
        ],
        'target_name': 'target'
    }
}

def load_dataset(dataset_name):
    """
    加载指定的数据集
    
    Args:
        dataset_name: 数据集配置名称
        
    Returns:
        x: 特征数据
        y: 目标数据
    """
    if dataset_name not in DATASET_CONFIGS:
        raise ValueError(f"未知的数据集名称: {dataset_name}")
    
    config = DATASET_CONFIGS[dataset_name]
    data = pd.read_excel(config['file_path'])
    
    x = data.iloc[:, config['x_columns']]
    y = data.iloc[:, config['y_column']]
    
    # 设置列名
    x.columns = config['feature_names']
    y.name = config['target_name']
    
    return x, y

# 选择要使用的数据集
# dataset_name = 'soil_acid_9features'              # 土壤反酸数据：64个样本，9个特征(包含delta_ph)，目标 105_day_ph
# dataset_name = 'soil_acid_8features_original'     # 土壤反酸数据：64个样本，8个特征，目标 delta_ph
dataset_name = 'soil_acid_8features'                # 土壤反酸数据：60个样本（去除异常点），8个特征，目标 delta_ph
# dataset_name = 'soil_acid_6features'              # 土壤反酸数据：34个样本，6个特征，目标 delta_ph
# dataset_name = 'acidity_reduce'                   # 精准降酸数据：54个样本，5个特征，目标是1/b
# dataset_name = 'acidity_reduce_new'                   # 精准降酸数据(数据更新)：54个样本，5个特征，目标是1/b
x, y = load_dataset(dataset_name)

print("特征数据:")
print(x)
print("\n目标数据:")
print(y)

## 数据集划分
Xtrain, Xtest, Ytrain, Ytest = train_test_split(x, y, test_size=0.2, random_state=42)

## 模型
# 随机森林回归模型
rfc = RandomForestRegressor(random_state=1)
# XGBR
XGB = XGBR(random_state=1)
# GBSTR
GBST = GBSTR(random_state=1)
# KNN
KNN = KNeighborsRegressor(n_neighbors=2)

# 增量训练：每次增加10%的训练数据
increment = 0.1  # 每次增加的比例
train_sizes = np.arange(0.1, 1.1, increment)  # 从0.1到1.0的训练集大小
r2_scores_rfc = []  
r2_scores_xgb = []  
r2_scores_gbst = []  
r2_scores_knn = []  # 用来记录KNN的r2_score


# 对于每种训练集大小，训练模型并记录r2_score
for size in train_sizes[:10]:
    # 计算当前训练集的大小
    current_size = int(size * len(Xtrain))
    
    # RandomForestRegressor
    rfc.fit(Xtrain[:current_size], Ytrain[:current_size])
    # XGBRegressor
    XGB.fit(Xtrain[:current_size], Ytrain[:current_size])
    # GBST
    GBST.fit(Xtrain[:current_size], Ytrain[:current_size])
    # KNN
    KNN.fit(Xtrain[:current_size], Ytrain[:current_size])

    # r2_score
    y_pred_rfc = rfc.predict(Xtest)
    score_rfc = r2_score(Ytest, y_pred_rfc)
    r2_scores_rfc.append(score_rfc)
    
    # XGBRegressor的r2_score
    y_pred_xgb = XGB.predict(Xtest)
    score_xgb = r2_score(Ytest, y_pred_xgb)
    r2_scores_xgb.append(score_xgb)

    # GBST的r2_score
    y_pred_gbst = GBST.predict(Xtest)
    score_gbst = r2_score(Ytest, y_pred_gbst)
    r2_scores_gbst.append(score_gbst)

    # KNN的r2_score
    y_pred_knn = KNN.predict(Xtest)
    score_knn = r2_score(Ytest, y_pred_knn)
    r2_scores_knn.append(score_knn)

    # 输出当前的训练进度与r2评分
    print(f"Training with {size * 100:.2f}% of the data:")
    print(f"  - Random Forest R2 score: {score_rfc}")
    print(f"  - XGB R2 score: {score_xgb}")
    print(f"  - GBST R2 score: {score_gbst}")
    print(f"  - KNN R2 score: {score_knn}")

# 绘制R2评分随训练数据大小变化的图形
import matplotlib.pyplot as plt

plt.plot(train_sizes * 100, r2_scores_rfc, marker='o', label='Random Forest')
plt.plot(train_sizes * 100, r2_scores_xgb, marker='x', label='XGBoost')
plt.plot(train_sizes * 100, r2_scores_gbst, marker='s', label='Gradient Boosting')
# plt.plot(train_sizes * 100, r2_scores_knn, marker='^', label='KNN')


plt.xlabel('Training data size (%)')
plt.ylabel('R2 Score')
plt.title('Model Performance with Incremental Data')
plt.legend()
plt.grid(True)
plt.show()

# 打印JavaScript中需要的数据格式
print("X轴数据（训练数据大小）:", [f"{int(size * 100)}%" for size in train_sizes])
print("Random Forest R2分数:", r2_scores_rfc)
print("XGBoost R2分数:", r2_scores_xgb)
print("Gradient Boosting R2分数:", r2_scores_gbst)



y_pred = rfc.predict(Xtest)  # 使用任意一个模型，这里以随机森林为例
residuals = Ytest - y_pred

plt.scatter(Ytest, y_pred, color='blue', alpha=0.5)
plt.plot([min(Ytest), max(Ytest)], [min(Ytest), max(Ytest)], color='red', linestyle='--')  # 对角线 y=x
plt.xlabel('True Values')
plt.ylabel('Predicted Values')
plt.title('True vs Predicted Values')
plt.grid(True)
plt.show()
# 生成 scatterData
scatter_data = [[float(true), float(pred)] for true, pred in zip(Ytest, y_pred)]
# 打印 scatterData（可直接复制到 JavaScript 代码中）
print("scatterData = ", scatter_data)

# # 保存 X_test 和 Y_test 为 CSV 文件
# X_test_df = pd.DataFrame(Xtest)
# Y_test_df = pd.DataFrame(Ytest)

# # 将 X_test 和 Y_test 保存为 CSV 文件，方便之后加载
# X_test_df.to_csv('X_test_reflux.csv', index=False)
# Y_test_df.to_csv('Y_test_reflux.csv', index=False)

# # 输出提示信息
# print("X_test 和 Y_test 已保存为 'X_test_reduce.csv' 和 'Y_test_reduce.csv'")



# 选择后半部分数据
# import matplotlib.pyplot as plt

# # 选择后半部分数据
# half_index = len(train_sizes) // 2

# # 绘制后半部分的数据
# plt.plot(train_sizes[half_index:] * 100, r2_scores_rfc[half_index:], marker='o', label='Random Forest')
# plt.plot(train_sizes[half_index:] * 100, r2_scores_xgb[half_index:], marker='x', label='XGBoost')
# plt.plot(train_sizes[half_index:] * 100, r2_scores_gbst[half_index:], marker='s', label='Gradient Boosting')
# plt.plot(train_sizes[half_index:] * 100, r2_scores_knn[half_index:], marker='^', label='KNN')

# plt.xlabel('Training data size (%)')
# plt.ylabel('R2 Score')
# plt.title('Model Performance with Incremental Data (Second Half)')
# plt.legend()
# plt.grid(True)
# plt.show()