generate label for mysql

main_spider.py

@@ -17,7 +17,6 @@ import lightgbm as lgb
 from scipy.stats import randint as sp_randint
 from scipy.stats import uniform as sp_uniform
 from sklearn.model_selection import RandomizedSearchCV, train_test_split
-import lightgbm as lgb
 from sklearn.metrics import roc_auc_score, accuracy_score
 
 
@@ -55,17 +54,17 @@ class LightGBM(object):
         self.flag = flag
         self.dt = dt
 
-    def read_data(self):
+    def read_data(self, path):
         """
         Read data from local
         :return:
         """
-        path = "data/train_data/spider_data_240401.json"
         df = pd.read_json(path)
         df = df.dropna(subset=['label'])
         labels = df['label']
         temp = sorted(labels)
         yc = temp[int(len(temp) * 0.7)]
+        print("阈值", yc)
         labels = [0 if i < yc else 1 for i in labels]
         features = df.drop("label", axis=1)
         for key in self.float_columns:
@@ -75,7 +74,8 @@ class LightGBM(object):
         return features, labels
 
     def best_params(self):
-        X, y = self.read_data()
+        path = "data/train_data/spider_data_240401.json"
+        X, y = self.read_data(path)
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
 
         lgbm = lgb.LGBMClassifier(objective='binary')
@@ -118,7 +118,8 @@ class LightGBM(object):
         Load dataset
         :return:
         """
-        x, y = self.read_data()
+        path = "data/train_data/spider_data_240401.json"
+        x, y = self.read_data(path)
         train_size = int(len(x) * self.split_c)
         X_train, X_test = x[:train_size], x[train_size:]
         Y_train, Y_test = y[:train_size], y[train_size:]
@@ -150,23 +151,12 @@ class LightGBM(object):
         评估模型性能
         :return:
         """
-        fw = open("summary_tag_03{}_spider.txt".format(self.dt), "a+", encoding="utf-8")
-        # 测试数据
-        with open("data/produce_data/x_data_total_return_predict_{}_spider.json".format(self.dt)) as f1:
-            x_list = json.loads(f1.read())
-
-        # 测试 label
-        with open("data/produce_data/y_data_total_return_predict_{}_spider.json".format(self.dt)) as f2:
-            Y_test = json.loads(f2.read())
-
-        Y_test = [0 if i <= 19 else 1 for i in Y_test]
-        X_test = pd.DataFrame(x_list, columns=self.my_c)
-        for key in self.str_columns:
-            X_test[key] = self.label_encoder.fit_transform(X_test[key])
-        for key in self.float_columns:
-            X_test[key] = pd.to_numeric(X_test[key], errors="coerce")
+        fw = open("result/summary_{}.json".format(dt), "a+", encoding="utf-8")
+        path = 'data/predict_data/predict_{}.json'.format(dt)
+        x, y = self.read_data(path)
+        Y_test = [0 if i <= 19 else 1 for i in y]
         bst = lgb.Booster(model_file=self.model)
-        y_pred = bst.predict(X_test, num_iteration=bst.best_iteration)
+        y_pred = bst.predict(x, num_iteration=bst.best_iteration)
         temp = sorted(list(y_pred))
         yuzhi = temp[int(len(temp) * 0.7) - 1]
         y_pred_binary = [0 if i <= yuzhi else 1 for i in list(y_pred)]
@@ -212,7 +202,7 @@ if __name__ == "__main__":
         L.train_model()
     elif i == 2:
         f = "predict"
-        dt = int(input("输入日期， 16-21:\n"))
+        dt = int(input("输入日期， 20240316-21:\n"))
         L = LightGBM(flag=f, dt=dt)
         L.evaluate_model()
         L.feature_importance()