model_script/vov/vov_h24_analyse.py

import numpy as np
import pandas as pd
from scipy.optimize import minimize
from sklearn.metrics import r2_score
from sklearn.model_selection import train_test_split
import pickle

all_feature_names = ["1_vovh0", "2_vovh0", "2_vovh1", "3_vovh0", "3_vovh1", "3_vovh2", "4_vovh0", "4_vovh1", "4_vovh2",
                     "4_vovh3", "6_vovh0", "6_vovh1", "6_vovh6", "12_vovh0", "12_vovh1", "12_vovh12", "24_vovh0",
                     "24_vovh1", "24_vovh2", "24_vovh3", "24_vovh6", "24_vovh12", "24_vovh24", "48_vovh0", "48_vovh1",
                     "48_vovh2", "48_vovh3", "48_vovh6", "48_vovh12", "48_vovh24", "48_vovh48", "1_vovd0", "2_vovd0",
                     "3_vovd0", "4_vovd0", "5_vovd0", "2_vovd1", "3_vovd1", "4_vovd1", "5_vovd1", "3_vovd2", "4_vovd2",
                     "5_vovd2", "1_vovh_分母", "1_vovh0分子", "2_vovh_分母", "2_vovh0分子", "2_vovh1分子",
                     "3_vovh_分母", "3_vovh0分子", "3_vovh1分子", "3_vovh2分子", "4_vovh_分母", "4_vovh0分子",
                     "4_vovh1分子", "4_vovh2分子", "4_vovh3分子", "6_vovh_分母", "6_vovh0分子", "6_vovh1分子",
                     "6_vovh6分子", "12_vovh_分母", "12_vovh0分子", "12_vovh1分子", "12_vovh12分子", "24_vovh_分母",
                     "24_vovh0分子", "24_vovh1分子", "24_vovh2分子", "24_vovh3分子", "24_vovh6分子", "24_vovh12分子",
                     "24_vovh24分子", "48_vovh_分母", "48_vovh0分子", "48_vovh1分子", "48_vovh2分子", "48_vovh3分子",
                     "48_vovh6分子", "48_vovh12分子", "48_vovh24分子", "48_vovh48分子", "1_vovd0_分母", "1_vovd0_分子",
                     "2_vovd0_分母", "2_vovd0_分子", "3_vovd0_分母", "3_vovd0_分子", "4_vovd0_分母", "4_vovd0_分子",
                     "5_vovd0_分母", "5_vovd0_分子", "2_vovd1_分母", "2_vovd1_分子", "3_vovd1_分母", "3_vovd1_分子",
                     "4_vovd1_分母", "4_vovd1_分子", "5_vovd1_分母", "5_vovd1_分子", "3_vovd2_分母", "3_vovd2_分子",
                     "4_vovd2_分母", "4_vovd2_分子", "5_vovd2_分母", "5_vovd2_分子"]

# feature_names = ["1_vovh0",
#                  "2_vovh0", "2_vovh1",
#                  "3_vovh0", "3_vovh1", "3_vovh2",
#                  "4_vovh0", "4_vovh1", "4_vovh2", "4_vovh3",
#                  "6_vovh0", "6_vovh1", "6_vovh6",
#                  "12_vovh0", "12_vovh1", "12_vovh12",
#                  "24_vovh0", "24_vovh1", "24_vovh2", "24_vovh3", "24_vovh6", "24_vovh12", "24_vovh24",
#                  "48_vovh0", "48_vovh1", "48_vovh2", "48_vovh3", "48_vovh6", "48_vovh12", "48_vovh24", "48_vovh48",
#                  "1_vovd0", "2_vovd0", "3_vovd0",
#                  "2_vovd1", "3_vovd1"
#                  ]

feature_names = ["1_vovh0",
                 "2_vovh0", "2_vovh1",
                 "3_vovh1", "3_vovh2",
                 "4_vovh1", "4_vovh3",
                 "6_vovh1", "6_vovh6",
                 "12_vovh1", "12_vovh12",
                 "24_vovh1", "24_vovh2", "24_vovh3", "24_vovh6", "24_vovh12", "24_vovh24",
                 "48_vovh1", "48_vovh2", "48_vovh3", "48_vovh6", "48_vovh12", "48_vovh24", "48_vovh48",
                 "1_vovd0",
                 "2_vovd1", "3_vovd1"
                 ]

dt_list = ['20241014', '20241015', '20241016']
hh_list = ["00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12",
           "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"]

pd.set_option('display.max_rows', None)  # 显示所有行
pd.set_option('display.max_columns', None)  # 显示所有列


# 1. 加载数据
def load_data(filepath: str) -> pd.DataFrame:
    return pd.read_csv(filepath)


# 2. 数据预处理
def preprocess_data(df, features, target):
    df_sorted = df.sort_values(by=target, ascending=False)
    x = df_sorted[features]
    y = df_sorted[target]

    top_k = df_sorted.head(100)

    return x, y, top_k


# 3. 计算相关系数
def calculate_correlations(df, features, target):
    correlations = {}
    for feature in features:
        # 删除 target 或 feature 列中任一为空的行
        valid_data = df[[target, feature]].dropna()

        # 如果没有有效数据，相关系数设为 0
        if len(valid_data) == 0:
            correlations[feature] = 0
        else:
            # 计算相关系数
            corr = valid_data[target].corr(valid_data[feature])
            correlations[feature] = corr if not np.isnan(corr) else 0

    # 转换为 Series 并按绝对值大小排序
    corr_series = pd.Series(correlations).abs().sort_values(ascending=False)
    return corr_series


# 4. 定义动态加权和函数
def dynamic_weighted_sum(features, weights):
    valid_features = ~np.isnan(features)
    if np.sum(valid_features) == 0:
        return np.nan
    normalized_weights = weights[valid_features] / np.sum(weights[valid_features])
    return np.sum(features[valid_features] * normalized_weights)


# 5. 定义损失函数
def mse_loss(y_true, y_pred):
    valid = ~np.isnan(y_true) & ~np.isnan(y_pred)
    return np.mean((y_true[valid] - y_pred[valid]) ** 2)


# 6. 定义目标函数
def objective(weights, X, y_true):
    y_pred = np.array([dynamic_weighted_sum(x, weights) for x in X.values])
    return mse_loss(y_true, y_pred)


# 7. 搜索最佳权重
def find_best_weights(X, y, initial_weights):
    result = minimize(objective, initial_weights, args=(X, y), method='Nelder-Mead')
    return result.x


# 8. 评估模型
def evaluate_model(X, y, weights):
    y_pred = np.array([dynamic_weighted_sum(x, weights) for x in X.values])
    valid = ~np.isnan(y) & ~np.isnan(y_pred)
    r2 = r2_score(y[valid], y_pred[valid])
    mse = mse_loss(y, y_pred)
    return r2, mse


# 9. 保存模型
def save_model(weights, features, file_path):
    model = {
        'weights': weights,
        'features': features,
    }
    with open(file_path, 'wb') as f:
        pickle.dump(model, f)


# 10. 加载模型
def load_model(file_path):
    with open(file_path, 'rb') as f:
        model = pickle.load(f)
    return model['weights'], model['features']


def single_dt_handle(dt, df: pd.DataFrame):
    x, y, top_key = preprocess_data(df, feature_names, "vovh24")
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
    correl = calculate_correlations(top_key, feature_names, "vovh24")
    print(f"{dt}   Feature Correlations: ")
    print(correl.head(5))

    initial_weights = correl[feature_names].values
    best_weights = find_best_weights(x_train, y_train, initial_weights)
    # 评估模型
    r2_train, mse_train = evaluate_model(x_train, y_train, best_weights)
    r2_test, mse_test = evaluate_model(x_test, y_test, best_weights)

    print(f"\nTrain R² Score: {r2_train:.4f}, MSE: {mse_train:.4f}")
    print(f"Test R² Score: {r2_test:.4f}, MSE: {mse_test:.4f}")

    # 输出特征重要性
    print("\nFeature importance:")
    for feature, weight in zip(feature_names, best_weights):
        print(f"{feature}: {weight:.4f}")

    # 保存模型
    save_model(pd.Series(best_weights, index=feature_names), feature_names,
               '/Users/zhao/Desktop/vov/model/vovh24_linear_weighted_model.pkl')

    # 测试加载模型
    loaded_weights, loaded_features = load_model('/Users/zhao/Desktop/vov/model/vovh24_linear_weighted_model.pkl')
    print("\nLoaded model weights:")
    for feature, weight in loaded_weights.items():
        print(f"{feature}: {weight:.4f}")


def _main():
    df_dict = {}
    for dt in dt_list:
        for hh in hh_list:
            key = f"{dt}{hh}"
            df = load_data(f"/Users/zhao/Desktop/vov/{key}.csv")
            df_dict[key] = df

    for key in df_dict:
        single_dt_handle(key, df_dict[key])
        return


if __name__ == '__main__':
    _main()