仅通过标题tag 分析全部数据

main.py

@@ -1,17 +1,20 @@
+"""
+仅使用标题信息
+"""
 import os
 import sys
-import json
-import optuna
 
-from sklearn.linear_model import LogisticRegression
+from sklearn.preprocessing import LabelEncoder
 
 sys.path.append(os.getcwd())
 
-import numpy as np
 import pandas as pd
 import lightgbm as lgb
-from sklearn.preprocessing import LabelEncoder
-from sklearn.metrics import accuracy_score
+
+from scipy.stats import randint as sp_randint
+from scipy.stats import uniform as sp_uniform
+from sklearn.model_selection import RandomizedSearchCV, train_test_split
+from sklearn.metrics import roc_auc_score, accuracy_score
 
 
 class LightGBM(object):
@@ -22,137 +25,102 @@ class LightGBM(object):
     def __init__(self, flag, dt):
         self.label_encoder = LabelEncoder()
         self.my_c = [
-            "uid",
-            "type",
-            "channel",
-            "fans",
-            "view_count_user_30days",
-            "share_count_user_30days",
-            "return_count_user_30days",
-            "rov_user",
-            "str_user",
-            "out_user_id",
-            "mode",
-            "out_play_cnt",
-            "out_like_cnt",
-            "out_share_cnt",
-            "out_collection_cnt",
             "tag1",
             "tag2",
-            "tag3"
-        ]
-        self.str_columns = ["uid", "type", "channel", "mode", "out_user_id", "tag1", "tag2", "tag3"]
-        self.float_columns = [
-            "fans",
-            "view_count_user_30days",
-            "share_count_user_30days",
-            "return_count_user_30days",
-            "rov_user",
-            "str_user",
-            "out_play_cnt",
-            "out_like_cnt",
-            "out_share_cnt",
-            "out_collection_cnt",
+            "tag3",
+            "tag4"
         ]
-        self.split_c = 0.999
+        self.str_columns = ["tag1", "tag2", "tag3", "tag4"]
+        self.split_c = 0.7
         self.yc = 0.8
-        self.model = "lightgbm_0326.bin"
+        self.model = "models/lightgbm_0408_all_tags.bin"
         self.flag = flag
         self.dt = dt
 
-    def bays_params(self, trial):
-        """
-        Bayesian parameters for
-        :return: best parameters
-        """
-        # 定义搜索空间
-        param = {
-            'objective': 'binary',
-            'metric': 'binary_logloss',
-            'verbosity': -1,
-            'boosting_type': 'gbdt',
-            'num_leaves': trial.suggest_int('num_leaves', 20, 40),
-            'learning_rate': trial.suggest_loguniform('learning_rate', 1e-8, 1.0),
-            'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
-            'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
-            'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
-            'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
-            "num_threads": 16,  # 线程数量
-        }
-        X_train, X_test = self.generate_x_data()
-        Y_train, Y_test = self.generate_y_data()
-        train_data = lgb.Dataset(
-            X_train,
-            label=Y_train,
-            categorical_feature=["uid", "type", "channel", "mode", "out_user_id", "tag1", "tag2", "tag3"],
-        )
-        test_data = lgb.Dataset(X_test, label=Y_test, reference=train_data)
-        gbm = lgb.train(param, train_data, num_boost_round=100, valid_sets=[test_data])
-        preds = gbm.predict(X_test)
-        pred_labels = np.rint(preds)
-        accuracy = accuracy_score(Y_test, pred_labels)
-        return accuracy
-
-    def generate_x_data(self):
+    def read_data(self, path, yc=None):
         """
-        Generate data for feature engineering
+        Read data from local
         :return:
         """
-        with open("data/produce_data/x_data_total_return_{}_{}.json".format(self.flag, self.dt)) as f1:
-            x_list = json.loads(f1.read())
-        index_t = int(len(x_list) * self.split_c)
-        X_train = pd.DataFrame(x_list[:index_t], columns=self.my_c)
+        df = pd.read_json(path)
+        df = df.dropna(subset=['label'])  # 把 label 为空的删掉
+        df = df.drop(subset=['tag1', 'tag2', 'tag3', 'tag4'])  # 把 tag 为空的数据也删掉
+        labels = df['label']
+        features = df.drop('label', axis=1)
         for key in self.str_columns:
-            X_train[key] = self.label_encoder.fit_transform(X_train[key])
-        for key in self.float_columns:
-            X_train[key] = pd.to_numeric(X_train[key], errors="coerce")
-        X_test = pd.DataFrame(x_list[index_t:], columns=self.my_c)
-        for key in self.str_columns:
-            X_test[key] = self.label_encoder.fit_transform(X_test[key])
-        for key in self.float_columns:
-            X_test[key] = pd.to_numeric(X_test[key], errors="coerce")
-        return X_train, X_test
+            features[key] = self.label_encoder.fit_transform(features[key])
+        return features, labels, df
 
-    def generate_y_data(self):
+    def best_params(self):
         """
-        Generate data for label
-        :return:
+        find best params for lightgbm
         """
-        with open("data/produce_data/y_data_total_return_{}_{}.json".format(self.flag, self.dt)) as f2:
-            y_list = json.loads(f2.read())
-        index_t = int(len(y_list) * self.split_c)
-        temp = sorted(y_list)
-        yuzhi = temp[int(len(temp) * self.yc) - 1]
-        print("阈值是: {}".format(yuzhi))
-        y__list = [0 if i <= yuzhi else 1 for i in y_list]
-        y_train = np.array(y__list[:index_t])
-        y_test = np.array(y__list[index_t:])
-        return y_train, y_test
+        path = "data/train_data/all_train_20240408.json"
+        X, y, ori_df = self.read_data(path)
+        print(len(list(y)))
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+        lgb_ = lgb.LGBMClassifier(objective='binary')
+
+        # 设置搜索的参数范围
+        param_dist = {
+            'num_leaves': sp_randint(20, 40),
+            'learning_rate': sp_uniform(0.001, 0.1),
+            'feature_fraction': sp_uniform(0.5, 0.9),
+            'bagging_fraction': sp_uniform(0.5, 0.9),
+            'bagging_freq': sp_randint(1, 10),
+            'min_child_samples': sp_randint(5, 100),
+        }
+
+        # 定义 RandomizedSearchCV
+        rsearch = RandomizedSearchCV(
+            estimator=lgb_,
+            param_distributions=param_dist,
+            n_iter=100,
+            cv=3,
+            scoring='roc_auc',
+            random_state=42, verbose=2
+        )
+
+        # 开始搜索
+        rsearch.fit(X_train, y_train)
+
+        # 打印最佳参数和对应的AUC得分
+        print("Best parameters found: ", rsearch.best_params_)
+        print("Best AUC found: ", rsearch.best_score_)
+
+        # 使用最佳参数在测试集上的表现
+        best_model = rsearch.best_estimator_
+        y_pred = best_model.predict_proba(X_test)[:, 1]
+        auc = roc_auc_score(y_test, y_pred)
+        print("AUC on test set: ", auc)
 
     def train_model(self):
         """
         Load dataset
         :return:
         """
-        X_train, X_test = self.generate_x_data()
-        Y_train, Y_test = self.generate_y_data()
+        path = "data/train_data/user_train_20240408.json"
+        x, y, ori_df = self.read_data(path)
+        train_size = int(len(x) * self.split_c)
+        X_train, X_test = x[:train_size], x[train_size:]
+        Y_train, Y_test = y[:train_size], y[train_size:]
         train_data = lgb.Dataset(
             X_train,
             label=Y_train,
-            categorical_feature=["uid", "type", "channel", "mode", "out_user_id", "tag1", "tag2", "tag3"],
+            categorical_feature=["channel", "mode", "out_user_id", "tag1", "tag2", "tag3"],
         )
         test_data = lgb.Dataset(X_test, label=Y_test, reference=train_data)
         params = {
-            "objective": "binary",  # 指定二分类任务
-            "metric": "binary_logloss",  # 评估指标为二分类的log损失
-            "num_leaves": 36,  # 叶子节点数
-            "learning_rate":  0.08479152931388902,  # 学习率
-            "bagging_fraction": 0.6588121592044218,  # 建树的样本采样比例
-            "feature_fraction": 0.4572757903437793,  # 建树的特征选择比例
-            "bagging_freq": 2,  # k 意味着每 k 次迭代执行bagging
-            "num_threads": 16,  # 线程数量
-            "mini_child_samples": 71
+            'bagging_fraction': 0.9323330736797192,
+            'bagging_freq': 1,
+            'feature_fraction': 0.8390650729441467,
+            'learning_rate': 0.07595782999760721,
+            'min_child_samples': 93,
+            'num_leaves': 36,
+            'num_threads': 16
         }
+
         # 训练模型
         num_round = 100
         print("开始训练......")
@@ -165,33 +133,24 @@ class LightGBM(object):
         评估模型性能
         :return:
         """
-        fw = open("summary_tag_03{}.txt".format(self.dt), "a+", encoding="utf-8")
-        # 测试数据
-        with open("data/produce_data/x_data_total_return_predict_{}.json".format(self.dt)) as f1:
-            x_list = json.loads(f1.read())
-
-        # 测试 label
-        with open("data/produce_data/y_data_total_return_predict_{}.json".format(self.dt)) as f2:
-            Y_test = json.loads(f2.read())
-
-        Y_test = [0 if i <= 27 else 1 for i in Y_test]
-        X_test = pd.DataFrame(x_list, columns=self.my_c)
-        for key in self.str_columns:
-            X_test[key] = self.label_encoder.fit_transform(X_test[key])
-        for key in self.float_columns:
-            X_test[key] = pd.to_numeric(X_test[key], errors="coerce")
+        fw = open("result/summary_{}.txt".format(dt), "a+", encoding="utf-8")
+        path = 'data/predict_data/predict_{}.json'.format(dt)
+        x, y, ori_df = self.read_data(path, yc=6)
+        true_label_df = pd.DataFrame(list(y), columns=['ture_label'])
         bst = lgb.Booster(model_file=self.model)
-        y_pred = bst.predict(X_test, num_iteration=bst.best_iteration)
-        temp = sorted(list(y_pred))
-        yuzhi = temp[int(len(temp) * 0.7) - 1]
-        y_pred_binary = [0 if i <= yuzhi else 1 for i in list(y_pred)]
-        # 转换为二进制输出
+        y_pred = bst.predict(x, num_iteration=bst.best_iteration)
+        pred_score_df = pd.DataFrame(list(y_pred), columns=['pred_score'])
+        # temp = sorted(list(y_pred))
+        # yuzhi = temp[int(len(temp) * 0.9) - 1]
+        y_pred_binary = [0 if i <= 0.169541 else 1 for i in list(y_pred)]
+        pred_label_df = pd.DataFrame(list(y_pred_binary), columns=['pred_label'])
         score_list = []
         for index, item in enumerate(list(y_pred)):
-            real_label = Y_test[index]
+            real_label = y[index]
             score = item
             prid_label = y_pred_binary[index]
-            print(real_label, "\t", prid_label, "\t", score)
+            if score < 0.169541:
+                print(real_label, "\t", prid_label, "\t", score)
             fw.write("{}\t{}\t{}\n".format(real_label, prid_label, score))
             score_list.append(score)
         print("预测样本总量: {}".format(len(score_list)))
@@ -199,9 +158,14 @@ class LightGBM(object):
         print("统计 score 信息")
         print(data_series.describe())
         # 评估模型
-        accuracy = accuracy_score(Y_test, y_pred_binary)
+        accuracy = accuracy_score(y, y_pred_binary)
         print(f"Accuracy: {accuracy}")
         fw.close()
+        # 水平合并
+        df_concatenated = pd.concat([ori_df, true_label_df, pred_score_df, pred_label_df], axis=1)
+        # for key in self.str_columns:
+        #     df_concatenated[key] = [self.label_mapping[key][i] for i in df_concatenated[key]]
+        df_concatenated.to_excel("data/predict_data/spider_predict_result_{}.xlsx".format(dt), index=False)
 
     def feature_importance(self):
         """
@@ -227,13 +191,10 @@ if __name__ == "__main__":
         L.train_model()
     elif i == 2:
         f = "predict"
-        dt = int(input("输入日期， 16-21:\n"))
+        dt = int(input("输入日期， 20240316-21:\n"))
         L = LightGBM(flag=f, dt=dt)
         L.evaluate_model()
-    # study = optuna.create_study(direction='maximize')
-    # study.optimize(L.bays_params, n_trials=100)
-    # print('Number of finished trials:', len(study.trials))
-    # print('Best trial:', study.best_trial.params)
-    # L.train_model()
-    # L.evaluate_model()
-    # L.feature_importance()
+        L.feature_importance()
+    elif i == 3:
+        L = LightGBM("train", "whole")
+        L.best_params()