{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "import pandas as pd\n", "from utils import send_msg_to_feishu" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "133\n", " Unnamed: 0 feature_name data_count 0_count 0_rate \\\n", "0 0 stage_one_retrn 319586 290427 0.908760 \n", "1 1 stage_two_retrn 319586 305954 0.957345 \n", "2 2 stage_four_retrn 319586 314081 0.982775 \n", "3 3 stage_three_retrn 319586 311375 0.974307 \n", "4 4 stage_two_return_added 319586 290970 0.910459 \n", ".. ... ... ... ... ... \n", "128 128 videocategory6 319586 316051 0.988939 \n", "129 129 videocategory1 319586 314240 0.983272 \n", "130 130 videocategory49 319586 309823 0.969451 \n", "131 131 videocategory45 319586 319323 0.999177 \n", "132 132 videocategory7 319586 316632 0.990757 \n", "\n", " mean_0.25 mean_0.5 mean_0.75 mean_1 var_0.25 ... \\\n", "0 0.000000 0.000000 0.000000 29.053141 0.000000 ... \n", "1 0.000000 0.000000 0.000000 18.705926 0.000000 ... \n", "2 0.000000 0.000000 0.000000 8.048241 0.000000 ... \n", "3 0.000000 0.000000 0.000000 12.135716 0.000000 ... \n", "4 -43.642152 -21.821076 -14.547445 -10.347215 233092.343014 ... \n", ".. ... ... ... ... ... ... \n", "128 0.000000 0.000000 0.000000 0.011061 0.000000 ... \n", "129 0.000000 0.000000 0.000000 0.016728 0.000000 ... \n", "130 0.000000 0.000000 0.000000 0.030549 0.000000 ... \n", "131 0.000000 0.000000 0.000000 0.000823 0.000000 ... \n", "132 0.000000 0.000000 0.000000 0.009243 0.000000 ... \n", "\n", " non_zero_mean_0.75 non_zero_mean_1 non_zero_var_0.25 non_zero_var_0.5 \\\n", "0 7.139735 318.425769 1.459960e-01 2.504982e+00 \n", "1 21.390416 438.538146 1.029324e+00 2.315790e+01 \n", "2 53.258658 467.230699 7.457763e+00 2.379205e+02 \n", "3 40.048709 472.342589 2.707221e+00 9.688260e+01 \n", "4 -162.274612 -115.558604 2.398539e+06 1.253440e+06 \n", ".. ... ... ... ... \n", "128 1.000000 1.000000 0.000000e+00 0.000000e+00 \n", "129 1.000000 1.000000 0.000000e+00 0.000000e+00 \n", "130 1.000000 1.000000 0.000000e+00 0.000000e+00 \n", "131 1.000000 1.000000 0.000000e+00 0.000000e+00 \n", "132 1.000000 1.000000 0.000000e+00 0.000000e+00 \n", "\n", " non_zero_var_0.75 non_zero_var_1 non_zero_std_0.25 non_zero_std_0.5 \\\n", "0 70.237904 4.695676e+06 0.382094 1.582714 \n", "1 788.731488 4.739503e+06 1.014556 4.812266 \n", "2 3900.825806 3.281403e+06 2.730890 15.424672 \n", "3 2589.586846 3.965567e+06 1.645364 9.842896 \n", "4 848453.781924 6.494582e+05 1548.721605 1119.571313 \n", ".. ... ... ... ... \n", "128 0.000000 0.000000e+00 0.000000 0.000000 \n", "129 0.000000 0.000000e+00 0.000000 0.000000 \n", "130 0.000000 0.000000e+00 0.000000 0.000000 \n", "131 0.000000 0.000000e+00 0.000000 0.000000 \n", "132 0.000000 0.000000e+00 0.000000 0.000000 \n", "\n", " non_zero_std_0.75 non_zero_std_1 \n", "0 8.380806 2166.950796 \n", "1 28.084364 2177.040037 \n", "2 62.456591 1811.464241 \n", "3 50.887983 1991.373137 \n", "4 921.115510 805.889696 \n", ".. ... ... \n", "128 0.000000 0.000000 \n", "129 0.000000 0.000000 \n", "130 0.000000 0.000000 \n", "131 0.000000 0.000000 \n", "132 0.000000 0.000000 \n", "\n", "[133 rows x 29 columns]\n" ] } ], "source": [ "train_file_path_20220218 = './data/train_data_monitor_20220218.csv'\n", "df_train_20220218 = pd.read_csv(train_file_path_20220218)\n", "print(len(df_train_20220218))\n", "print(df_train_20220218)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Unnamed: 0 feature_name data_count 0_count 0_rate \\\n", "0 0 stage_one_retrn 319586 290427 0.908760 \n", "1 1 stage_two_retrn 319586 305954 0.957345 \n", "2 2 stage_four_retrn 319586 314081 0.982775 \n", "3 3 stage_three_retrn 319586 311375 0.974307 \n", "4 4 stage_two_return_added 319586 290970 0.910459 \n", "5 5 stage_three_return_added 319586 306146 0.957946 \n", "6 6 stage_four_return_added 319586 311477 0.974627 \n", "7 7 stage_two_return_ratio 319586 290970 0.910459 \n", "8 8 stage_four_return_ratio 319586 311477 0.974627 \n", "9 9 stage_three_return_ratio 319586 306146 0.957946 \n", "\n", " mean_0.25 mean_0.5 mean_0.75 mean_1 var_0.25 ... \\\n", "0 0.000000 0.000000 0.000000 29.053141 0.000000 ... \n", "1 0.000000 0.000000 0.000000 18.705926 0.000000 ... \n", "2 0.000000 0.000000 0.000000 8.048241 0.000000 ... \n", "3 0.000000 0.000000 0.000000 12.135716 0.000000 ... \n", "4 -43.642152 -21.821076 -14.547445 -10.347215 233092.343014 ... \n", "5 -27.887705 -13.943853 -9.295941 -6.570210 99191.736492 ... \n", "6 -17.560559 -8.780280 -5.853544 -4.087476 43690.700442 ... \n", "7 -inf -inf -inf -inf NaN ... \n", "8 -inf -inf -inf -inf NaN ... \n", "9 -inf -inf -inf -inf NaN ... \n", "\n", " non_zero_mean_0.75 non_zero_mean_1 non_zero_var_0.25 non_zero_var_0.5 \\\n", "0 7.139735 318.425769 1.459960e-01 2.504982e+00 \n", "1 21.390416 438.538146 1.029324e+00 2.315790e+01 \n", "2 53.258658 467.230699 7.457763e+00 2.379205e+02 \n", "3 40.048709 472.342589 2.707221e+00 9.688260e+01 \n", "4 -162.274612 -115.558604 2.398539e+06 1.253440e+06 \n", "5 -220.820454 -156.231176 1.976463e+06 1.080257e+06 \n", "6 -230.444919 -161.092613 1.321688e+06 7.500555e+05 \n", "7 -inf -inf NaN NaN \n", "8 -inf -inf NaN NaN \n", "9 -inf -inf NaN NaN \n", "\n", " non_zero_var_0.75 non_zero_var_1 non_zero_std_0.25 non_zero_std_0.5 \\\n", "0 70.237904 4.695676e+06 0.382094 1.582714 \n", "1 788.731488 4.739503e+06 1.014556 4.812266 \n", "2 3900.825806 3.281403e+06 2.730890 15.424672 \n", "3 2589.586846 3.965567e+06 1.645364 9.842896 \n", "4 848453.781924 6.494582e+05 1548.721605 1119.571313 \n", "5 743546.359590 5.877715e+05 1405.867249 1039.354097 \n", "6 524894.428157 4.317385e+05 1149.646842 866.057452 \n", "7 NaN NaN NaN NaN \n", "8 NaN NaN NaN NaN \n", "9 NaN NaN NaN NaN \n", "\n", " non_zero_std_0.75 non_zero_std_1 \n", "0 8.380806 2166.950796 \n", "1 28.084364 2177.040037 \n", "2 62.456591 1811.464241 \n", "3 50.887983 1991.373137 \n", "4 921.115510 805.889696 \n", "5 862.291343 766.662541 \n", "6 724.495982 657.068119 \n", "7 NaN NaN \n", "8 NaN NaN \n", "9 NaN NaN \n", "\n", "[10 rows x 29 columns]\n" ] } ], "source": [ "print(df_train_20220218.head(10))" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Unnamed: 0 feature_name data_count 0_count 0_rate \\\n", "0 0 stage_one_retrn 308675 279287 0.904793 \n", "1 1 stage_two_retrn 308675 294930 0.955471 \n", "2 2 stage_four_retrn 308675 303106 0.981958 \n", "3 3 stage_three_retrn 308675 300388 0.973153 \n", "4 4 stage_two_return_added 308675 279831 0.906555 \n", "5 5 stage_three_return_added 308675 295119 0.956083 \n", "6 6 stage_four_return_added 308675 300487 0.973474 \n", "7 7 stage_two_return_ratio 308675 279831 0.906555 \n", "8 8 stage_four_return_ratio 308675 300487 0.973474 \n", "9 9 stage_three_return_ratio 308675 295119 0.956083 \n", "\n", " mean_0.25 mean_0.5 mean_0.75 mean_1 var_0.25 ... \\\n", "0 0.000000 0.000000 0.000000 30.684182 0.000000 ... \n", "1 0.000000 0.000000 0.000000 19.802935 0.000000 ... \n", "2 0.000000 0.000000 0.000000 8.577209 0.000000 ... \n", "3 0.000000 0.000000 0.000000 12.867560 0.000000 ... \n", "4 -45.941933 -22.970966 -15.313978 -10.881247 246586.644725 ... \n", "5 -29.466509 -14.733254 -9.822170 -6.935375 105535.713482 ... \n", "6 -18.534321 -9.267160 -6.178107 -4.290351 46148.960740 ... \n", "7 -inf -inf -inf -inf NaN ... \n", "8 -inf -inf -inf -inf NaN ... \n", "9 -inf -inf -inf -inf NaN ... \n", "\n", " non_zero_mean_0.75 non_zero_mean_1 non_zero_var_0.25 non_zero_var_0.5 \\\n", "0 7.207377 322.289370 1.471082e-01 2.535730e+00 \n", "1 21.718983 444.719607 1.050753e+00 2.365002e+01 \n", "2 54.492219 475.412103 7.670916e+00 2.505415e+02 \n", "3 40.826094 479.292144 2.763735e+00 1.006501e+02 \n", "4 -163.690302 -116.446020 2.431524e+06 1.270892e+06 \n", "5 -223.428600 -157.920994 2.012587e+06 1.100527e+06 \n", "6 -232.626343 -161.739619 1.332624e+06 7.570373e+05 \n", "7 -inf -inf NaN NaN \n", "8 -inf -inf NaN NaN \n", "9 -inf -inf NaN NaN \n", "\n", " non_zero_var_0.75 non_zero_var_1 non_zero_std_0.25 non_zero_std_0.5 \\\n", "0 72.045508 4.778534e+06 0.383547 1.592398 \n", "1 817.447948 4.818667e+06 1.025062 4.863129 \n", "2 4099.125209 3.347202e+06 2.769642 15.828503 \n", "3 2697.035876 4.017723e+06 1.662449 10.032454 \n", "4 860314.590137 6.589329e+05 1559.334576 1127.338453 \n", "5 757621.619017 5.989774e+05 1418.656878 1049.060053 \n", "6 530023.047128 4.410375e+05 1154.393260 870.078894 \n", "7 NaN NaN NaN NaN \n", "8 NaN NaN NaN NaN \n", "9 NaN NaN NaN NaN \n", "\n", " non_zero_std_0.75 non_zero_std_1 \n", "0 8.487963 2185.985905 \n", "1 28.591047 2195.146262 \n", "2 64.024411 1829.535988 \n", "3 51.932994 2004.425852 \n", "4 927.531450 811.746798 \n", "5 870.414625 773.936274 \n", "6 728.026818 664.106568 \n", "7 NaN NaN \n", "8 NaN NaN \n", "9 NaN NaN \n", "\n", "[10 rows x 29 columns]\n" ] } ], "source": [ "train_file_path_20220219 = './data/train_data_monitor_20220219.csv'\n", "df_train_20220219 = pd.read_csv(train_file_path_20220219)\n", "print(df_train_20220219.head(10))" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 var_1 non_zero_mean_0.25 non_zero_mean_0.5 \\\n", "0 0.004365 0.056140 0.061938 0.001468 0.004284 \n", "1 0.001957 0.058645 0.061745 0.008027 0.009626 \n", "2 0.000831 0.065725 0.069314 0.008695 0.021213 \n", "3 0.001185 0.060305 0.059503 0.010793 0.016691 \n", "4 0.004288 -0.051611 0.058754 -0.008783 -0.008744 \n", "5 0.001944 -0.055579 0.064221 -0.011894 -0.011860 \n", "6 0.001183 -0.049633 0.067102 -0.008996 -0.009480 \n", "7 0.004288 NaN NaN NaN NaN \n", "8 0.001183 NaN NaN NaN NaN \n", "9 0.001944 NaN NaN NaN NaN \n", "\n", " non_zero_mean_0.75 non_zero_mean_1 non_zero_var_0.25 non_zero_var_0.5 \\\n", "0 0.009474 0.012133 0.007618 0.012275 \n", "1 0.015361 0.014096 0.020818 0.021251 \n", "2 0.023162 0.017510 0.028581 0.053047 \n", "3 0.019411 0.014713 0.020875 0.038888 \n", "4 -0.008724 -0.007679 0.013752 0.013923 \n", "5 -0.011811 -0.010816 0.018277 0.018764 \n", "6 -0.009466 -0.004016 0.008274 0.009308 \n", "7 NaN NaN NaN NaN \n", "8 NaN NaN NaN NaN \n", "9 NaN NaN NaN NaN \n", "\n", " non_zero_var_0.75 non_zero_var_1 feature_name \n", "0 0.025735 0.017646 stage_one_retrn \n", "1 0.036408 0.016703 stage_two_retrn \n", "2 0.050835 0.020052 stage_four_retrn \n", "3 0.041493 0.013152 stage_three_retrn \n", "4 0.013979 0.014589 stage_two_return_added \n", "5 0.018930 0.019065 stage_three_return_added \n", "6 0.009771 0.021539 stage_four_return_added \n", "7 NaN NaN stage_two_return_ratio \n", "8 NaN NaN stage_four_return_ratio \n", "9 NaN NaN stage_three_return_ratio \n" ] } ], "source": [ "feature_top10_20220219 = df_train_20220219.head(10)['feature_name']\n", "monitor_list = ['0_rate', 'mean_1', 'var_1', \n", " 'non_zero_mean_0.25', 'non_zero_mean_0.5', 'non_zero_mean_0.75', 'non_zero_mean_1',\n", " 'non_zero_var_0.25', 'non_zero_var_0.5', 'non_zero_var_0.75', 'non_zero_var_1']\n", "diff_20220218 = df_train_20220218.loc[df_train_20220218['feature_name'].isin(feature_top10_20220219)][monitor_list]\n", "diff_20220219 = df_train_20220219.loc[df_train_20220219['feature_name'].isin(feature_top10_20220219)][monitor_list]\n", "diff = abs(diff_20220219-diff_20220218)/diff_20220218\n", "diff['feature_name'] = feature_top10_20220219\n", "print(diff)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 30, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "def cal_diff(yesterday_file, today_file):\n", " df_train_yesterday = pd.read_csv(yesterday_file)\n", " df_train_today = pd.read_csv(today_file)\n", "\n", " feature_top10_today = df_train_today.head(10)['feature_name']\n", " monitor_list = ['0_rate', 'mean_1', 'var_1', \n", " 'non_zero_mean_0.25', 'non_zero_mean_0.5', 'non_zero_mean_0.75', 'non_zero_mean_1',\n", " 'non_zero_var_0.25', 'non_zero_var_0.5', 'non_zero_var_0.75', 'non_zero_var_1']\n", " df_yesterday = df_train_yesterday.loc[df_train_yesterday['feature_name'].isin(feature_top10_today)][monitor_list]\n", " df_today = df_train_today.loc[df_train_today['feature_name'].isin(feature_top10_today)][monitor_list]\n", " diff = abs(df_today - df_yesterday) / df_yesterday\n", " diff['feature_name'] = feature_top10_today\n", " print(diff)" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 var_1 non_zero_mean_0.25 non_zero_mean_0.5 \\\n", "0 0.005002 0.057325 0.071410 0.000330 0.000139 \n", "1 0.002172 0.061239 0.110287 0.000205 0.001382 \n", "2 0.000856 0.076648 0.379979 0.002546 0.002176 \n", "3 0.001281 0.066219 0.211559 0.006632 0.003632 \n", "4 0.004924 -0.050201 0.103066 -0.008135 -0.007913 \n", "5 0.002149 -0.052000 0.083131 -0.009921 -0.009605 \n", "6 0.001273 -0.045370 0.046743 -0.001187 -0.001560 \n", "7 0.004924 NaN NaN NaN NaN \n", "8 0.001273 NaN NaN NaN NaN \n", "9 0.002149 NaN NaN NaN NaN \n", "\n", " non_zero_mean_0.75 non_zero_mean_1 non_zero_var_0.25 non_zero_var_0.5 \\\n", "0 0.000849 0.009346 0.001695 0.001666 \n", "1 0.003023 0.013982 0.001716 0.002233 \n", "2 0.011587 0.028726 0.002981 0.003943 \n", "3 0.007318 0.018900 0.021120 0.000492 \n", "4 -0.007917 -0.002322 0.032027 0.031292 \n", "5 -0.009624 -0.004975 0.019300 0.019249 \n", "6 -0.001606 -0.001296 0.006401 0.005717 \n", "7 NaN NaN NaN NaN \n", "8 NaN NaN NaN NaN \n", "9 NaN NaN NaN NaN \n", "\n", " non_zero_var_0.75 non_zero_var_1 feature_name \n", "0 0.001764 0.022971 stage_one_retrn \n", "1 0.008437 0.062215 stage_two_retrn \n", "2 0.036278 0.335875 stage_four_retrn \n", "3 0.019741 0.164520 stage_three_retrn \n", "4 0.031083 0.053767 stage_two_return_added \n", "5 0.019300 0.035786 stage_three_return_added \n", "6 0.005394 0.000239 stage_four_return_added \n", "7 NaN NaN stage_two_return_ratio \n", "8 NaN NaN stage_four_return_ratio \n", "9 NaN NaN stage_three_return_ratio \n" ] } ], "source": [ "file_20220219 = './data/train_data_monitor_20220219.csv'\n", "file_20220220 = './data/train_data_monitor_20220220.csv'\n", "cal_diff(yesterday_file=file_20220219, today_file=file_20220220)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 var_1 non_zero_mean_0.25 non_zero_mean_0.5 \\\n", "0 0.005572 0.063716 0.206425 0.000137 0.002426 \n", "1 0.002313 0.072997 0.489410 0.007128 0.002891 \n", "2 0.000955 0.110413 1.329780 0.001553 0.009829 \n", "3 0.001368 0.090244 0.962106 0.002813 0.002741 \n", "4 0.005481 -0.046648 0.111135 -0.003523 -0.003452 \n", "5 0.002273 -0.040563 0.062435 -0.001561 -0.001249 \n", "6 0.001349 -0.048717 0.061383 -0.000450 -0.000628 \n", "7 0.005481 NaN NaN NaN NaN \n", "8 0.020957 NaN NaN NaN NaN \n", "9 0.017709 NaN NaN NaN NaN \n", "\n", " non_zero_mean_0.75 non_zero_mean_1 non_zero_var_0.25 non_zero_var_0.5 \\\n", "0 0.004514 0.012775 0.000702 0.008503 \n", "1 0.002557 0.024518 0.018237 0.000353 \n", "2 0.010414 0.057912 0.010849 0.018093 \n", "3 0.003311 0.040968 0.000559 0.006996 \n", "4 -0.003437 -0.003670 0.022244 0.021602 \n", "5 -0.001314 -0.006318 0.001523 0.001560 \n", "6 -0.000693 -0.001422 0.019594 0.017566 \n", "7 NaN NaN NaN NaN \n", "8 NaN NaN NaN NaN \n", "9 NaN NaN NaN NaN \n", "\n", " non_zero_var_0.75 non_zero_var_1 feature_name \n", "0 0.013407 0.151157 stage_one_retrn \n", "1 0.005393 0.436318 stage_two_retrn \n", "2 0.020078 1.277436 stage_four_retrn \n", "3 0.006591 0.912625 stage_three_retrn \n", "4 0.021383 0.058964 stage_two_return_added \n", "5 0.001633 0.015707 stage_three_return_added \n", "6 0.016856 0.014209 stage_four_return_added \n", "7 NaN NaN stage_two_return_ratio \n", "8 NaN NaN stage_three_return_ratio \n", "9 NaN NaN stage_four_return_ratio \n" ] } ], "source": [ "file_20220220 = './data/train_data_monitor_20220220.csv'\n", "file_20220221 = './data/train_data_monitor_20220221.csv'\n", "cal_diff(yesterday_file=file_20220220, today_file=file_20220221)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 2, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [ "def cal_diff2(yesterday_file, today_file, feature_top=-1):\n", " df_train_yesterday = pd.read_csv(yesterday_file, index_col='feature_name')\n", " df_train_today = pd.read_csv(today_file, index_col='feature_name')\n", " \n", " # 计算数据偏移量\n", " feature_top_today = df_train_today.index.values[:feature_top]\n", " # print(feature_top10_today)\n", " monitor_list = ['0_rate', 'mean_1', 'var_1', \n", " 'non_zero_mean_0.25', 'non_zero_mean_0.5', 'non_zero_mean_0.75', 'non_zero_mean_1',\n", " 'non_zero_var_0.25', 'non_zero_var_0.5', 'non_zero_var_0.75', 'non_zero_var_1']\n", " df_yesterday = df_train_yesterday.loc[feature_top_today, monitor_list]\n", " # print(df_yesterday)\n", " df_today = df_train_today.loc[feature_top_today, monitor_list]\n", " # print(df_today)\n", " diff = abs(df_today - df_yesterday) / df_yesterday\n", " \n", " # 获取偏移超过 10%的特征\n", " offset_feature_list = []\n", " for column in diff.columns.to_list():\n", " temp_df = diff.loc[diff[column]>=0.1]\n", " if not temp_df.empty:\n", " offset_feature_list.append(temp_df)\n", " offset_feature_df = pd.concat(offset_feature_list)\n", " # 去重\n", " offset_feature_df.drop_duplicates(inplace=True)\n", " \n", " return offset_feature_df" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 \\\n", "feature_name \n", "stage_four_retrn 0.000955 0.110413 \n", "videocategory2 0.010831 0.111064 \n", "videocategory10 0.001226 0.159772 \n", "videocategory6 0.001242 0.137218 \n", "videocategory8 0.003189 0.126149 \n", "videocategory12 0.001257 0.149417 \n", "videocategory45 0.000074 0.101004 \n", "stage_one_retrn 0.005572 0.063716 \n", "stage_two_retrn 0.002313 0.072997 \n", "stage_three_retrn 0.001368 0.090244 \n", "stage_two_return_added 0.005481 -0.046648 \n", "four_up_return_day1_view_day1_return_count 0.000415 0.082808 \n", "four_up_return_div_three_return_day1 0.001412 0.048325 \n", "day1playcount 0.026017 0.067653 \n", "todyviewcount_rank NaN 0.058987 \n", "usercategory8 0.011316 0.039797 \n", "usercategory49 0.002495 0.031561 \n", "day3viewcount 0.009367 0.058399 \n", "day7viewcount 0.023594 0.061726 \n", "usercategory12 0.013911 0.020124 \n", "usercategory6 0.001753 0.006368 \n", "usercategory45 0.043287 0.056887 \n", "usercategory10 0.026096 0.017610 \n", "usercategory85 0.035717 0.003181 \n", "usercategory1 0.019829 0.009141 \n", "four_up_return_day3_view_day3_return_count 0.000957 0.068258 \n", "usercategory7 0.050857 0.025389 \n", "all_return_day1_return_count 0.007139 0.062472 \n", "usercategory9 0.030080 0.025230 \n", "three_return_day1_return_count 0.007083 0.062605 \n", "four_up_return_day7_return_count 0.001563 0.056201 \n", "all_return_day14_return_count 0.008692 0.052830 \n", "todyviewcount NaN 0.054649 \n", "usercategory11 0.044032 0.022018 \n", "usercategory4 0.006735 0.022629 \n", "four_up_return_day14_return_count 0.001364 0.046675 \n", "four_up_return_day1_return_count 0.001428 0.062099 \n", "four_up_return_day3_return_count 0.001563 0.052683 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.000957 0.053065 \n", "day3ctr 0.032330 0.015493 \n", "day7ctr 0.034606 0.018660 \n", "\n", " var_1 \\\n", "feature_name \n", "stage_four_retrn 1.329780 \n", "videocategory2 0.101436 \n", "videocategory10 0.158741 \n", "videocategory6 0.136146 \n", "videocategory8 0.123362 \n", "videocategory12 0.148348 \n", "videocategory45 0.100937 \n", "stage_one_retrn 0.206425 \n", "stage_two_retrn 0.489410 \n", "stage_three_retrn 0.962106 \n", "stage_two_return_added 0.111135 \n", "four_up_return_day1_view_day1_return_count 0.256227 \n", "four_up_return_div_three_return_day1 0.504287 \n", "day1playcount 0.108513 \n", "todyviewcount_rank 0.172902 \n", "usercategory8 0.040935 \n", "usercategory49 0.034497 \n", "day3viewcount 0.075310 \n", "day7viewcount 0.081014 \n", "usercategory12 0.044976 \n", "usercategory6 0.009099 \n", "usercategory45 0.068209 \n", "usercategory10 0.018733 \n", "usercategory85 0.013926 \n", "usercategory1 0.009071 \n", "four_up_return_day3_view_day3_return_count 0.054460 \n", "usercategory7 0.042184 \n", "all_return_day1_return_count 0.063919 \n", "usercategory9 0.040968 \n", "three_return_day1_return_count 0.072129 \n", "four_up_return_day7_return_count 0.049085 \n", "all_return_day14_return_count 0.061986 \n", "todyviewcount 0.076444 \n", "usercategory11 0.006161 \n", "usercategory4 0.033476 \n", "four_up_return_day14_return_count 0.033090 \n", "four_up_return_day1_return_count 0.045627 \n", "four_up_return_day3_return_count 0.039399 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.049843 \n", "day3ctr 0.059119 \n", "day7ctr 0.050853 \n", "\n", " non_zero_mean_0.25 \\\n", "feature_name \n", "stage_four_retrn 0.001553 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.000137 \n", "stage_two_retrn 0.007128 \n", "stage_three_retrn 0.002813 \n", "stage_two_return_added -0.003523 \n", "four_up_return_day1_view_day1_return_count 0.002859 \n", "four_up_return_div_three_return_day1 0.016367 \n", "day1playcount 0.008248 \n", "todyviewcount_rank 0.034262 \n", "usercategory8 0.144831 \n", "usercategory49 0.145017 \n", "day3viewcount 0.378388 \n", "day7viewcount 0.369795 \n", "usercategory12 0.027332 \n", "usercategory6 0.078629 \n", "usercategory45 0.055002 \n", "usercategory10 0.049888 \n", "usercategory85 0.014034 \n", "usercategory1 0.009520 \n", "four_up_return_day3_view_day3_return_count 0.036363 \n", "usercategory7 0.031458 \n", "all_return_day1_return_count 0.006241 \n", "usercategory9 0.062283 \n", "three_return_day1_return_count 0.007088 \n", "four_up_return_day7_return_count 0.032509 \n", "all_return_day14_return_count 0.018621 \n", "todyviewcount 0.000000 \n", "usercategory11 0.004512 \n", "usercategory4 0.039397 \n", "four_up_return_day14_return_count 0.035060 \n", "four_up_return_day1_return_count 0.005887 \n", "four_up_return_day3_return_count 0.024137 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.021652 \n", "day3ctr 0.007768 \n", "day7ctr 0.012601 \n", "\n", " non_zero_mean_0.5 \\\n", "feature_name \n", "stage_four_retrn 0.009829 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.002426 \n", "stage_two_retrn 0.002891 \n", "stage_three_retrn 0.002741 \n", "stage_two_return_added -0.003452 \n", "four_up_return_day1_view_day1_return_count 0.002179 \n", "four_up_return_div_three_return_day1 0.019310 \n", "day1playcount 0.027277 \n", "todyviewcount_rank 0.034269 \n", "usercategory8 0.135420 \n", "usercategory49 0.115789 \n", "day3viewcount 0.121670 \n", "day7viewcount 0.104674 \n", "usercategory12 0.055497 \n", "usercategory6 0.042390 \n", "usercategory45 0.054150 \n", "usercategory10 0.012099 \n", "usercategory85 0.047790 \n", "usercategory1 0.020109 \n", "four_up_return_day3_view_day3_return_count 0.029564 \n", "usercategory7 0.058902 \n", "all_return_day1_return_count 0.006836 \n", "usercategory9 0.095772 \n", "three_return_day1_return_count 0.008354 \n", "four_up_return_day7_return_count 0.040166 \n", "all_return_day14_return_count 0.023581 \n", "todyviewcount 0.054714 \n", "usercategory11 0.043522 \n", "usercategory4 0.043823 \n", "four_up_return_day14_return_count 0.054255 \n", "four_up_return_day1_return_count 0.024934 \n", "four_up_return_day3_return_count 0.033559 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.018049 \n", "day3ctr 0.003958 \n", "day7ctr 0.007342 \n", "\n", " non_zero_mean_0.75 \\\n", "feature_name \n", "stage_four_retrn 0.010414 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.004514 \n", "stage_two_retrn 0.002557 \n", "stage_three_retrn 0.003311 \n", "stage_two_return_added -0.003437 \n", "four_up_return_day1_view_day1_return_count 0.000774 \n", "four_up_return_div_three_return_day1 0.016906 \n", "day1playcount 0.022199 \n", "todyviewcount_rank 0.034629 \n", "usercategory8 0.083957 \n", "usercategory49 0.065647 \n", "day3viewcount 0.078234 \n", "day7viewcount 0.066762 \n", "usercategory12 0.030142 \n", "usercategory6 0.013628 \n", "usercategory45 0.040219 \n", "usercategory10 0.003313 \n", "usercategory85 0.050542 \n", "usercategory1 0.000484 \n", "four_up_return_day3_view_day3_return_count 0.030331 \n", "usercategory7 0.041463 \n", "all_return_day1_return_count 0.012683 \n", "usercategory9 0.058713 \n", "three_return_day1_return_count 0.013850 \n", "four_up_return_day7_return_count 0.054221 \n", "all_return_day14_return_count 0.026809 \n", "todyviewcount 0.092298 \n", "usercategory11 0.035500 \n", "usercategory4 0.042675 \n", "four_up_return_day14_return_count 0.062747 \n", "four_up_return_day1_return_count 0.041597 \n", "four_up_return_day3_return_count 0.051668 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.014849 \n", "day3ctr 0.001072 \n", "day7ctr 0.003971 \n", "\n", " non_zero_mean_1 \\\n", "feature_name \n", "stage_four_retrn 0.057912 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.012775 \n", "stage_two_retrn 0.024518 \n", "stage_three_retrn 0.040968 \n", "stage_two_return_added -0.003670 \n", "four_up_return_day1_view_day1_return_count 0.025033 \n", "four_up_return_div_three_return_day1 0.017316 \n", "day1playcount 0.022780 \n", "todyviewcount_rank 0.058987 \n", "usercategory8 0.042837 \n", "usercategory49 0.032021 \n", "day3viewcount 0.052805 \n", "day7viewcount 0.050658 \n", "usercategory12 0.014068 \n", "usercategory6 0.005199 \n", "usercategory45 0.033315 \n", "usercategory10 0.008161 \n", "usercategory85 0.005018 \n", "usercategory1 0.005178 \n", "four_up_return_day3_view_day3_return_count 0.017286 \n", "usercategory7 0.032458 \n", "all_return_day1_return_count 0.026116 \n", "usercategory9 0.028566 \n", "three_return_day1_return_count 0.025082 \n", "four_up_return_day7_return_count 0.036746 \n", "all_return_day14_return_count 0.028682 \n", "todyviewcount 0.054649 \n", "usercategory11 0.008586 \n", "usercategory4 0.023515 \n", "four_up_return_day14_return_count 0.031379 \n", "four_up_return_day1_return_count 0.033956 \n", "four_up_return_day3_return_count 0.030638 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.002818 \n", "day3ctr 0.011206 \n", "day7ctr 0.007188 \n", "\n", " non_zero_var_0.25 \\\n", "feature_name \n", "stage_four_retrn 0.010849 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.000702 \n", "stage_two_retrn 0.018237 \n", "stage_three_retrn 0.000559 \n", "stage_two_return_added 0.022244 \n", "four_up_return_day1_view_day1_return_count 0.008815 \n", "four_up_return_div_three_return_day1 0.036767 \n", "day1playcount 0.012271 \n", "todyviewcount_rank 0.067377 \n", "usercategory8 0.379763 \n", "usercategory49 0.576893 \n", "day3viewcount 0.557694 \n", "day7viewcount 0.883487 \n", "usercategory12 0.130214 \n", "usercategory6 0.139570 \n", "usercategory45 0.427017 \n", "usercategory10 0.166045 \n", "usercategory85 0.169806 \n", "usercategory1 0.114492 \n", "four_up_return_day3_view_day3_return_count 0.102537 \n", "usercategory7 0.122846 \n", "all_return_day1_return_count 0.124121 \n", "usercategory9 0.209519 \n", "three_return_day1_return_count 0.645306 \n", "four_up_return_day7_return_count 0.143251 \n", "all_return_day14_return_count 0.134059 \n", "todyviewcount NaN \n", "usercategory11 0.009885 \n", "usercategory4 0.030476 \n", "four_up_return_day14_return_count 0.097179 \n", "four_up_return_day1_return_count 0.024356 \n", "four_up_return_day3_return_count 0.093090 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.019762 \n", "day3ctr 0.000536 \n", "day7ctr 0.001180 \n", "\n", " non_zero_var_0.5 \\\n", "feature_name \n", "stage_four_retrn 0.018093 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.008503 \n", "stage_two_retrn 0.000353 \n", "stage_three_retrn 0.006996 \n", "stage_two_return_added 0.021602 \n", "four_up_return_day1_view_day1_return_count 0.009560 \n", "four_up_return_div_three_return_day1 0.041840 \n", "day1playcount 0.091721 \n", "todyviewcount_rank 0.067377 \n", "usercategory8 0.209333 \n", "usercategory49 0.159956 \n", "day3viewcount 0.032570 \n", "day7viewcount 0.074300 \n", "usercategory12 0.152885 \n", "usercategory6 0.012217 \n", "usercategory45 0.131324 \n", "usercategory10 0.020356 \n", "usercategory85 0.117835 \n", "usercategory1 0.013757 \n", "four_up_return_day3_view_day3_return_count 0.060346 \n", "usercategory7 0.127237 \n", "all_return_day1_return_count 0.017365 \n", "usercategory9 0.183533 \n", "three_return_day1_return_count 0.027203 \n", "four_up_return_day7_return_count 0.089129 \n", "all_return_day14_return_count 0.059231 \n", "todyviewcount 0.234774 \n", "usercategory11 0.136667 \n", "usercategory4 0.106830 \n", "four_up_return_day14_return_count 0.121594 \n", "four_up_return_day1_return_count 0.085318 \n", "four_up_return_day3_return_count 0.079053 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.032172 \n", "day3ctr 0.011389 \n", "day7ctr 0.013882 \n", "\n", " non_zero_var_0.75 \\\n", "feature_name \n", "stage_four_retrn 0.020078 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.013407 \n", "stage_two_retrn 0.005393 \n", "stage_three_retrn 0.006591 \n", "stage_two_return_added 0.021383 \n", "four_up_return_day1_view_day1_return_count 0.006945 \n", "four_up_return_div_three_return_day1 0.032526 \n", "day1playcount 0.041539 \n", "todyviewcount_rank 0.069366 \n", "usercategory8 0.100997 \n", "usercategory49 0.066917 \n", "day3viewcount 0.033151 \n", "day7viewcount 0.059949 \n", "usercategory12 0.010870 \n", "usercategory6 0.022983 \n", "usercategory45 0.043284 \n", "usercategory10 0.004968 \n", "usercategory85 0.111731 \n", "usercategory1 0.018887 \n", "four_up_return_day3_view_day3_return_count 0.061611 \n", "usercategory7 0.060318 \n", "all_return_day1_return_count 0.033697 \n", "usercategory9 0.072907 \n", "three_return_day1_return_count 0.036757 \n", "four_up_return_day7_return_count 0.120658 \n", "all_return_day14_return_count 0.056375 \n", "todyviewcount 0.215959 \n", "usercategory11 0.051559 \n", "usercategory4 0.073398 \n", "four_up_return_day14_return_count 0.132839 \n", "four_up_return_day1_return_count 0.100219 \n", "four_up_return_day3_return_count 0.118647 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.025368 \n", "day3ctr 0.019178 \n", "day7ctr 0.019550 \n", "\n", " non_zero_var_1 \n", "feature_name \n", "stage_four_retrn 1.277436 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.151157 \n", "stage_two_retrn 0.436318 \n", "stage_three_retrn 0.912625 \n", "stage_two_return_added 0.058964 \n", "four_up_return_day1_view_day1_return_count 0.203595 \n", "four_up_return_div_three_return_day1 0.466700 \n", "day1playcount 0.062169 \n", "todyviewcount_rank 0.172902 \n", "usercategory8 0.040812 \n", "usercategory49 0.033942 \n", "day3viewcount 0.069517 \n", "day7viewcount 0.069719 \n", "usercategory12 0.040111 \n", "usercategory6 0.007871 \n", "usercategory45 0.045161 \n", "usercategory10 0.010546 \n", "usercategory85 0.007461 \n", "usercategory1 0.005524 \n", "four_up_return_day3_view_day3_return_count 0.003348 \n", "usercategory7 0.048708 \n", "all_return_day1_return_count 0.027357 \n", "usercategory9 0.043999 \n", "three_return_day1_return_count 0.034166 \n", "four_up_return_day7_return_count 0.029382 \n", "all_return_day14_return_count 0.037552 \n", "todyviewcount 0.076444 \n", "usercategory11 0.005929 \n", "usercategory4 0.034067 \n", "four_up_return_day14_return_count 0.017569 \n", "four_up_return_day1_return_count 0.017424 \n", "four_up_return_day3_return_count 0.017275 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.104871 \n", "day3ctr 0.114604 \n", "day7ctr 0.102625 \n" ] } ], "source": [ "file_20220221 = './data/train_data_monitor_20220221.csv'\n", "file_20220220 = './data/train_data_monitor_20220220.csv'\n", "diff = cal_diff2(yesterday_file=file_20220220, today_file=file_20220221, feature_top=-1)\n", "print(diff)\n", "# diff.to_csv('./data/diff_20220219_20220220.csv')" ] }, { "cell_type": "code", "execution_count": 74, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0_ratemean_1var_1non_zero_mean_0.25non_zero_mean_0.5non_zero_mean_0.75non_zero_mean_1non_zero_var_0.25non_zero_var_0.5non_zero_var_0.75non_zero_var_1
feature_name
stage_four_retrn0.0009550.1104131.3297800.0015530.0098290.0104140.0579120.0108490.0180930.0200781.277436
videocategory20.0108310.1110640.1014360.0000000.0000000.0000000.000000NaNNaNNaNNaN
videocategory100.0012260.1597720.1587410.0000000.0000000.0000000.000000NaNNaNNaNNaN
videocategory60.0012420.1372180.1361460.0000000.0000000.0000000.000000NaNNaNNaNNaN
videocategory80.0031890.1261490.1233620.0000000.0000000.0000000.000000NaNNaNNaNNaN
videocategory120.0012570.1494170.1483480.0000000.0000000.0000000.000000NaNNaNNaNNaN
videocategory450.0000740.1010040.1009370.0000000.0000000.0000000.000000NaNNaNNaNNaN
\n", "
" ], "text/plain": [ " 0_rate mean_1 var_1 non_zero_mean_0.25 \\\n", "feature_name \n", "stage_four_retrn 0.000955 0.110413 1.329780 0.001553 \n", "videocategory2 0.010831 0.111064 0.101436 0.000000 \n", "videocategory10 0.001226 0.159772 0.158741 0.000000 \n", "videocategory6 0.001242 0.137218 0.136146 0.000000 \n", "videocategory8 0.003189 0.126149 0.123362 0.000000 \n", "videocategory12 0.001257 0.149417 0.148348 0.000000 \n", "videocategory45 0.000074 0.101004 0.100937 0.000000 \n", "\n", " non_zero_mean_0.5 non_zero_mean_0.75 non_zero_mean_1 \\\n", "feature_name \n", "stage_four_retrn 0.009829 0.010414 0.057912 \n", "videocategory2 0.000000 0.000000 0.000000 \n", "videocategory10 0.000000 0.000000 0.000000 \n", "videocategory6 0.000000 0.000000 0.000000 \n", "videocategory8 0.000000 0.000000 0.000000 \n", "videocategory12 0.000000 0.000000 0.000000 \n", "videocategory45 0.000000 0.000000 0.000000 \n", "\n", " non_zero_var_0.25 non_zero_var_0.5 non_zero_var_0.75 \\\n", "feature_name \n", "stage_four_retrn 0.010849 0.018093 0.020078 \n", "videocategory2 NaN NaN NaN \n", "videocategory10 NaN NaN NaN \n", "videocategory6 NaN NaN NaN \n", "videocategory8 NaN NaN NaN \n", "videocategory12 NaN NaN NaN \n", "videocategory45 NaN NaN NaN \n", "\n", " non_zero_var_1 \n", "feature_name \n", "stage_four_retrn 1.277436 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN " ] }, "execution_count": 74, "metadata": {}, "output_type": "execute_result" } ], "source": [ "diff.loc[(diff['0_rate']>0.1) | \n", " ((diff['mean_1']>0.1) & (diff['var_1']>0.1)) | \n", " ((diff['non_zero_mean_1']>0.1) & (diff['non_zero_var_1']>0.1))]" ] }, { "cell_type": "code", "execution_count": 87, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 \\\n", "feature_name \n", "stage_four_retrn 0.000955 0.110413 \n", "videocategory2 0.010831 0.111064 \n", "videocategory10 0.001226 0.159772 \n", "videocategory6 0.001242 0.137218 \n", "videocategory8 0.003189 0.126149 \n", "videocategory12 0.001257 0.149417 \n", "videocategory45 0.000074 0.101004 \n", "stage_one_retrn 0.005572 0.063716 \n", "stage_two_retrn 0.002313 0.072997 \n", "stage_three_retrn 0.001368 0.090244 \n", "stage_two_return_added 0.005481 -0.046648 \n", "four_up_return_day1_view_day1_return_count 0.000415 0.082808 \n", "four_up_return_div_three_return_day1 0.001412 0.048325 \n", "day1playcount 0.026017 0.067653 \n", "todyviewcount_rank NaN 0.058987 \n", "usercategory8 0.011316 0.039797 \n", "usercategory49 0.002495 0.031561 \n", "day3viewcount 0.009367 0.058399 \n", "day7viewcount 0.023594 0.061726 \n", "usercategory12 0.013911 0.020124 \n", "usercategory6 0.001753 0.006368 \n", "usercategory45 0.043287 0.056887 \n", "usercategory10 0.026096 0.017610 \n", "usercategory85 0.035717 0.003181 \n", "usercategory1 0.019829 0.009141 \n", "four_up_return_day3_view_day3_return_count 0.000957 0.068258 \n", "usercategory7 0.050857 0.025389 \n", "all_return_day1_return_count 0.007139 0.062472 \n", "usercategory9 0.030080 0.025230 \n", "three_return_day1_return_count 0.007083 0.062605 \n", "four_up_return_day7_return_count 0.001563 0.056201 \n", "all_return_day14_return_count 0.008692 0.052830 \n", "todyviewcount NaN 0.054649 \n", "usercategory11 0.044032 0.022018 \n", "usercategory4 0.006735 0.022629 \n", "four_up_return_day14_return_count 0.001364 0.046675 \n", "four_up_return_day1_return_count 0.001428 0.062099 \n", "four_up_return_day3_return_count 0.001563 0.052683 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.000957 0.053065 \n", "day3ctr 0.032330 0.015493 \n", "day7ctr 0.034606 0.018660 \n", "\n", " var_1 \\\n", "feature_name \n", "stage_four_retrn 1.329780 \n", "videocategory2 0.101436 \n", "videocategory10 0.158741 \n", "videocategory6 0.136146 \n", "videocategory8 0.123362 \n", "videocategory12 0.148348 \n", "videocategory45 0.100937 \n", "stage_one_retrn 0.206425 \n", "stage_two_retrn 0.489410 \n", "stage_three_retrn 0.962106 \n", "stage_two_return_added 0.111135 \n", "four_up_return_day1_view_day1_return_count 0.256227 \n", "four_up_return_div_three_return_day1 0.504287 \n", "day1playcount 0.108513 \n", "todyviewcount_rank 0.172902 \n", "usercategory8 0.040935 \n", "usercategory49 0.034497 \n", "day3viewcount 0.075310 \n", "day7viewcount 0.081014 \n", "usercategory12 0.044976 \n", "usercategory6 0.009099 \n", "usercategory45 0.068209 \n", "usercategory10 0.018733 \n", "usercategory85 0.013926 \n", "usercategory1 0.009071 \n", "four_up_return_day3_view_day3_return_count 0.054460 \n", "usercategory7 0.042184 \n", "all_return_day1_return_count 0.063919 \n", "usercategory9 0.040968 \n", "three_return_day1_return_count 0.072129 \n", "four_up_return_day7_return_count 0.049085 \n", "all_return_day14_return_count 0.061986 \n", "todyviewcount 0.076444 \n", "usercategory11 0.006161 \n", "usercategory4 0.033476 \n", "four_up_return_day14_return_count 0.033090 \n", "four_up_return_day1_return_count 0.045627 \n", "four_up_return_day3_return_count 0.039399 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.049843 \n", "day3ctr 0.059119 \n", "day7ctr 0.050853 \n", "\n", " non_zero_mean_0.25 \\\n", "feature_name \n", "stage_four_retrn 0.001553 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.000137 \n", "stage_two_retrn 0.007128 \n", "stage_three_retrn 0.002813 \n", "stage_two_return_added -0.003523 \n", "four_up_return_day1_view_day1_return_count 0.002859 \n", "four_up_return_div_three_return_day1 0.016367 \n", "day1playcount 0.008248 \n", "todyviewcount_rank 0.034262 \n", "usercategory8 0.144831 \n", "usercategory49 0.145017 \n", "day3viewcount 0.378388 \n", "day7viewcount 0.369795 \n", "usercategory12 0.027332 \n", "usercategory6 0.078629 \n", "usercategory45 0.055002 \n", "usercategory10 0.049888 \n", "usercategory85 0.014034 \n", "usercategory1 0.009520 \n", "four_up_return_day3_view_day3_return_count 0.036363 \n", "usercategory7 0.031458 \n", "all_return_day1_return_count 0.006241 \n", "usercategory9 0.062283 \n", "three_return_day1_return_count 0.007088 \n", "four_up_return_day7_return_count 0.032509 \n", "all_return_day14_return_count 0.018621 \n", "todyviewcount 0.000000 \n", "usercategory11 0.004512 \n", "usercategory4 0.039397 \n", "four_up_return_day14_return_count 0.035060 \n", "four_up_return_day1_return_count 0.005887 \n", "four_up_return_day3_return_count 0.024137 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.021652 \n", "day3ctr 0.007768 \n", "day7ctr 0.012601 \n", "\n", " non_zero_mean_0.5 \\\n", "feature_name \n", "stage_four_retrn 0.009829 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.002426 \n", "stage_two_retrn 0.002891 \n", "stage_three_retrn 0.002741 \n", "stage_two_return_added -0.003452 \n", "four_up_return_day1_view_day1_return_count 0.002179 \n", "four_up_return_div_three_return_day1 0.019310 \n", "day1playcount 0.027277 \n", "todyviewcount_rank 0.034269 \n", "usercategory8 0.135420 \n", "usercategory49 0.115789 \n", "day3viewcount 0.121670 \n", "day7viewcount 0.104674 \n", "usercategory12 0.055497 \n", "usercategory6 0.042390 \n", "usercategory45 0.054150 \n", "usercategory10 0.012099 \n", "usercategory85 0.047790 \n", "usercategory1 0.020109 \n", "four_up_return_day3_view_day3_return_count 0.029564 \n", "usercategory7 0.058902 \n", "all_return_day1_return_count 0.006836 \n", "usercategory9 0.095772 \n", "three_return_day1_return_count 0.008354 \n", "four_up_return_day7_return_count 0.040166 \n", "all_return_day14_return_count 0.023581 \n", "todyviewcount 0.054714 \n", "usercategory11 0.043522 \n", "usercategory4 0.043823 \n", "four_up_return_day14_return_count 0.054255 \n", "four_up_return_day1_return_count 0.024934 \n", "four_up_return_day3_return_count 0.033559 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.018049 \n", "day3ctr 0.003958 \n", "day7ctr 0.007342 \n", "\n", " non_zero_mean_0.75 \\\n", "feature_name \n", "stage_four_retrn 0.010414 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.004514 \n", "stage_two_retrn 0.002557 \n", "stage_three_retrn 0.003311 \n", "stage_two_return_added -0.003437 \n", "four_up_return_day1_view_day1_return_count 0.000774 \n", "four_up_return_div_three_return_day1 0.016906 \n", "day1playcount 0.022199 \n", "todyviewcount_rank 0.034629 \n", "usercategory8 0.083957 \n", "usercategory49 0.065647 \n", "day3viewcount 0.078234 \n", "day7viewcount 0.066762 \n", "usercategory12 0.030142 \n", "usercategory6 0.013628 \n", "usercategory45 0.040219 \n", "usercategory10 0.003313 \n", "usercategory85 0.050542 \n", "usercategory1 0.000484 \n", "four_up_return_day3_view_day3_return_count 0.030331 \n", "usercategory7 0.041463 \n", "all_return_day1_return_count 0.012683 \n", "usercategory9 0.058713 \n", "three_return_day1_return_count 0.013850 \n", "four_up_return_day7_return_count 0.054221 \n", "all_return_day14_return_count 0.026809 \n", "todyviewcount 0.092298 \n", "usercategory11 0.035500 \n", "usercategory4 0.042675 \n", "four_up_return_day14_return_count 0.062747 \n", "four_up_return_day1_return_count 0.041597 \n", "four_up_return_day3_return_count 0.051668 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.014849 \n", "day3ctr 0.001072 \n", "day7ctr 0.003971 \n", "\n", " non_zero_mean_1 \\\n", "feature_name \n", "stage_four_retrn 0.057912 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.012775 \n", "stage_two_retrn 0.024518 \n", "stage_three_retrn 0.040968 \n", "stage_two_return_added -0.003670 \n", "four_up_return_day1_view_day1_return_count 0.025033 \n", "four_up_return_div_three_return_day1 0.017316 \n", "day1playcount 0.022780 \n", "todyviewcount_rank 0.058987 \n", "usercategory8 0.042837 \n", "usercategory49 0.032021 \n", "day3viewcount 0.052805 \n", "day7viewcount 0.050658 \n", "usercategory12 0.014068 \n", "usercategory6 0.005199 \n", "usercategory45 0.033315 \n", "usercategory10 0.008161 \n", "usercategory85 0.005018 \n", "usercategory1 0.005178 \n", "four_up_return_day3_view_day3_return_count 0.017286 \n", "usercategory7 0.032458 \n", "all_return_day1_return_count 0.026116 \n", "usercategory9 0.028566 \n", "three_return_day1_return_count 0.025082 \n", "four_up_return_day7_return_count 0.036746 \n", "all_return_day14_return_count 0.028682 \n", "todyviewcount 0.054649 \n", "usercategory11 0.008586 \n", "usercategory4 0.023515 \n", "four_up_return_day14_return_count 0.031379 \n", "four_up_return_day1_return_count 0.033956 \n", "four_up_return_day3_return_count 0.030638 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.002818 \n", "day3ctr 0.011206 \n", "day7ctr 0.007188 \n", "\n", " non_zero_var_0.25 \\\n", "feature_name \n", "stage_four_retrn 0.010849 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.000702 \n", "stage_two_retrn 0.018237 \n", "stage_three_retrn 0.000559 \n", "stage_two_return_added 0.022244 \n", "four_up_return_day1_view_day1_return_count 0.008815 \n", "four_up_return_div_three_return_day1 0.036767 \n", "day1playcount 0.012271 \n", "todyviewcount_rank 0.067377 \n", "usercategory8 0.379763 \n", "usercategory49 0.576893 \n", "day3viewcount 0.557694 \n", "day7viewcount 0.883487 \n", "usercategory12 0.130214 \n", "usercategory6 0.139570 \n", "usercategory45 0.427017 \n", "usercategory10 0.166045 \n", "usercategory85 0.169806 \n", "usercategory1 0.114492 \n", "four_up_return_day3_view_day3_return_count 0.102537 \n", "usercategory7 0.122846 \n", "all_return_day1_return_count 0.124121 \n", "usercategory9 0.209519 \n", "three_return_day1_return_count 0.645306 \n", "four_up_return_day7_return_count 0.143251 \n", "all_return_day14_return_count 0.134059 \n", "todyviewcount NaN \n", "usercategory11 0.009885 \n", "usercategory4 0.030476 \n", "four_up_return_day14_return_count 0.097179 \n", "four_up_return_day1_return_count 0.024356 \n", "four_up_return_day3_return_count 0.093090 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.019762 \n", "day3ctr 0.000536 \n", "day7ctr 0.001180 \n", "\n", " non_zero_var_0.5 \\\n", "feature_name \n", "stage_four_retrn 0.018093 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.008503 \n", "stage_two_retrn 0.000353 \n", "stage_three_retrn 0.006996 \n", "stage_two_return_added 0.021602 \n", "four_up_return_day1_view_day1_return_count 0.009560 \n", "four_up_return_div_three_return_day1 0.041840 \n", "day1playcount 0.091721 \n", "todyviewcount_rank 0.067377 \n", "usercategory8 0.209333 \n", "usercategory49 0.159956 \n", "day3viewcount 0.032570 \n", "day7viewcount 0.074300 \n", "usercategory12 0.152885 \n", "usercategory6 0.012217 \n", "usercategory45 0.131324 \n", "usercategory10 0.020356 \n", "usercategory85 0.117835 \n", "usercategory1 0.013757 \n", "four_up_return_day3_view_day3_return_count 0.060346 \n", "usercategory7 0.127237 \n", "all_return_day1_return_count 0.017365 \n", "usercategory9 0.183533 \n", "three_return_day1_return_count 0.027203 \n", "four_up_return_day7_return_count 0.089129 \n", "all_return_day14_return_count 0.059231 \n", "todyviewcount 0.234774 \n", "usercategory11 0.136667 \n", "usercategory4 0.106830 \n", "four_up_return_day14_return_count 0.121594 \n", "four_up_return_day1_return_count 0.085318 \n", "four_up_return_day3_return_count 0.079053 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.032172 \n", "day3ctr 0.011389 \n", "day7ctr 0.013882 \n", "\n", " non_zero_var_0.75 \\\n", "feature_name \n", "stage_four_retrn 0.020078 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.013407 \n", "stage_two_retrn 0.005393 \n", "stage_three_retrn 0.006591 \n", "stage_two_return_added 0.021383 \n", "four_up_return_day1_view_day1_return_count 0.006945 \n", "four_up_return_div_three_return_day1 0.032526 \n", "day1playcount 0.041539 \n", "todyviewcount_rank 0.069366 \n", "usercategory8 0.100997 \n", "usercategory49 0.066917 \n", "day3viewcount 0.033151 \n", "day7viewcount 0.059949 \n", "usercategory12 0.010870 \n", "usercategory6 0.022983 \n", "usercategory45 0.043284 \n", "usercategory10 0.004968 \n", "usercategory85 0.111731 \n", "usercategory1 0.018887 \n", "four_up_return_day3_view_day3_return_count 0.061611 \n", "usercategory7 0.060318 \n", "all_return_day1_return_count 0.033697 \n", "usercategory9 0.072907 \n", "three_return_day1_return_count 0.036757 \n", "four_up_return_day7_return_count 0.120658 \n", "all_return_day14_return_count 0.056375 \n", "todyviewcount 0.215959 \n", "usercategory11 0.051559 \n", "usercategory4 0.073398 \n", "four_up_return_day14_return_count 0.132839 \n", "four_up_return_day1_return_count 0.100219 \n", "four_up_return_day3_return_count 0.118647 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.025368 \n", "day3ctr 0.019178 \n", "day7ctr 0.019550 \n", "\n", " non_zero_var_1 \n", "feature_name \n", "stage_four_retrn 1.277436 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.151157 \n", "stage_two_retrn 0.436318 \n", "stage_three_retrn 0.912625 \n", "stage_two_return_added 0.058964 \n", "four_up_return_day1_view_day1_return_count 0.203595 \n", "four_up_return_div_three_return_day1 0.466700 \n", "day1playcount 0.062169 \n", "todyviewcount_rank 0.172902 \n", "usercategory8 0.040812 \n", "usercategory49 0.033942 \n", "day3viewcount 0.069517 \n", "day7viewcount 0.069719 \n", "usercategory12 0.040111 \n", "usercategory6 0.007871 \n", "usercategory45 0.045161 \n", "usercategory10 0.010546 \n", "usercategory85 0.007461 \n", "usercategory1 0.005524 \n", "four_up_return_day3_view_day3_return_count 0.003348 \n", "usercategory7 0.048708 \n", "all_return_day1_return_count 0.027357 \n", "usercategory9 0.043999 \n", "three_return_day1_return_count 0.034166 \n", "four_up_return_day7_return_count 0.029382 \n", "all_return_day14_return_count 0.037552 \n", "todyviewcount 0.076444 \n", "usercategory11 0.005929 \n", "usercategory4 0.034067 \n", "four_up_return_day14_return_count 0.017569 \n", "four_up_return_day1_return_count 0.017424 \n", "four_up_return_day3_return_count 0.017275 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.104871 \n", "day3ctr 0.114604 \n", "day7ctr 0.102625 \n" ] } ], "source": [ "offset_feature_list = []\n", "for column in diff.columns.to_list():\n", " temp_df = diff.loc[diff[column]>=0.1]\n", "# print(column)\n", "# print(temp_df)\n", " if not temp_df.empty:\n", " offset_feature_list.append(temp_df)\n", "offset_feature_df = pd.concat(offset_feature_list)\n", "offset_feature_df.drop_duplicates(inplace=True)\n", "print(offset_feature_df)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 \\\n", "feature_name \n", "stage_four_retrn 0.000955 0.110413 \n", "videocategory2 0.010831 0.111064 \n", "videocategory10 0.001226 0.159772 \n", "videocategory6 0.001242 0.137218 \n", "videocategory8 0.003189 0.126149 \n", "videocategory12 0.001257 0.149417 \n", "videocategory45 0.000074 0.101004 \n", "stage_one_retrn 0.005572 0.063716 \n", "stage_two_retrn 0.002313 0.072997 \n", "stage_three_retrn 0.001368 0.090244 \n", "stage_two_return_added 0.005481 -0.046648 \n", "four_up_return_day1_view_day1_return_count 0.000415 0.082808 \n", "four_up_return_div_three_return_day1 0.001412 0.048325 \n", "day1playcount 0.026017 0.067653 \n", "todyviewcount_rank NaN 0.058987 \n", "usercategory8 0.011316 0.039797 \n", "usercategory49 0.002495 0.031561 \n", "day3viewcount 0.009367 0.058399 \n", "day7viewcount 0.023594 0.061726 \n", "usercategory12 0.013911 0.020124 \n", "usercategory6 0.001753 0.006368 \n", "usercategory45 0.043287 0.056887 \n", "usercategory10 0.026096 0.017610 \n", "usercategory85 0.035717 0.003181 \n", "usercategory1 0.019829 0.009141 \n", "four_up_return_day3_view_day3_return_count 0.000957 0.068258 \n", "usercategory7 0.050857 0.025389 \n", "all_return_day1_return_count 0.007139 0.062472 \n", "usercategory9 0.030080 0.025230 \n", "three_return_day1_return_count 0.007083 0.062605 \n", "four_up_return_day7_return_count 0.001563 0.056201 \n", "all_return_day14_return_count 0.008692 0.052830 \n", "todyviewcount NaN 0.054649 \n", "usercategory11 0.044032 0.022018 \n", "usercategory4 0.006735 0.022629 \n", "four_up_return_day14_return_count 0.001364 0.046675 \n", "four_up_return_day1_return_count 0.001428 0.062099 \n", "four_up_return_day3_return_count 0.001563 0.052683 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.000957 0.053065 \n", "day3ctr 0.032330 0.015493 \n", "day7ctr 0.034606 0.018660 \n", "\n", " var_1 \\\n", "feature_name \n", "stage_four_retrn 1.329780 \n", "videocategory2 0.101436 \n", "videocategory10 0.158741 \n", "videocategory6 0.136146 \n", "videocategory8 0.123362 \n", "videocategory12 0.148348 \n", "videocategory45 0.100937 \n", "stage_one_retrn 0.206425 \n", "stage_two_retrn 0.489410 \n", "stage_three_retrn 0.962106 \n", "stage_two_return_added 0.111135 \n", "four_up_return_day1_view_day1_return_count 0.256227 \n", "four_up_return_div_three_return_day1 0.504287 \n", "day1playcount 0.108513 \n", "todyviewcount_rank 0.172902 \n", "usercategory8 0.040935 \n", "usercategory49 0.034497 \n", "day3viewcount 0.075310 \n", "day7viewcount 0.081014 \n", "usercategory12 0.044976 \n", "usercategory6 0.009099 \n", "usercategory45 0.068209 \n", "usercategory10 0.018733 \n", "usercategory85 0.013926 \n", "usercategory1 0.009071 \n", "four_up_return_day3_view_day3_return_count 0.054460 \n", "usercategory7 0.042184 \n", "all_return_day1_return_count 0.063919 \n", "usercategory9 0.040968 \n", "three_return_day1_return_count 0.072129 \n", "four_up_return_day7_return_count 0.049085 \n", "all_return_day14_return_count 0.061986 \n", "todyviewcount 0.076444 \n", "usercategory11 0.006161 \n", "usercategory4 0.033476 \n", "four_up_return_day14_return_count 0.033090 \n", "four_up_return_day1_return_count 0.045627 \n", "four_up_return_day3_return_count 0.039399 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.049843 \n", "day3ctr 0.059119 \n", "day7ctr 0.050853 \n", "\n", " non_zero_mean_0.25 \\\n", "feature_name \n", "stage_four_retrn 0.001553 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.000137 \n", "stage_two_retrn 0.007128 \n", "stage_three_retrn 0.002813 \n", "stage_two_return_added -0.003523 \n", "four_up_return_day1_view_day1_return_count 0.002859 \n", "four_up_return_div_three_return_day1 0.016367 \n", "day1playcount 0.008248 \n", "todyviewcount_rank 0.034262 \n", "usercategory8 0.144831 \n", "usercategory49 0.145017 \n", "day3viewcount 0.378388 \n", "day7viewcount 0.369795 \n", "usercategory12 0.027332 \n", "usercategory6 0.078629 \n", "usercategory45 0.055002 \n", "usercategory10 0.049888 \n", "usercategory85 0.014034 \n", "usercategory1 0.009520 \n", "four_up_return_day3_view_day3_return_count 0.036363 \n", "usercategory7 0.031458 \n", "all_return_day1_return_count 0.006241 \n", "usercategory9 0.062283 \n", "three_return_day1_return_count 0.007088 \n", "four_up_return_day7_return_count 0.032509 \n", "all_return_day14_return_count 0.018621 \n", "todyviewcount 0.000000 \n", "usercategory11 0.004512 \n", "usercategory4 0.039397 \n", "four_up_return_day14_return_count 0.035060 \n", "four_up_return_day1_return_count 0.005887 \n", "four_up_return_day3_return_count 0.024137 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.021652 \n", "day3ctr 0.007768 \n", "day7ctr 0.012601 \n", "\n", " non_zero_mean_0.5 \\\n", "feature_name \n", "stage_four_retrn 0.009829 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.002426 \n", "stage_two_retrn 0.002891 \n", "stage_three_retrn 0.002741 \n", "stage_two_return_added -0.003452 \n", "four_up_return_day1_view_day1_return_count 0.002179 \n", "four_up_return_div_three_return_day1 0.019310 \n", "day1playcount 0.027277 \n", "todyviewcount_rank 0.034269 \n", "usercategory8 0.135420 \n", "usercategory49 0.115789 \n", "day3viewcount 0.121670 \n", "day7viewcount 0.104674 \n", "usercategory12 0.055497 \n", "usercategory6 0.042390 \n", "usercategory45 0.054150 \n", "usercategory10 0.012099 \n", "usercategory85 0.047790 \n", "usercategory1 0.020109 \n", "four_up_return_day3_view_day3_return_count 0.029564 \n", "usercategory7 0.058902 \n", "all_return_day1_return_count 0.006836 \n", "usercategory9 0.095772 \n", "three_return_day1_return_count 0.008354 \n", "four_up_return_day7_return_count 0.040166 \n", "all_return_day14_return_count 0.023581 \n", "todyviewcount 0.054714 \n", "usercategory11 0.043522 \n", "usercategory4 0.043823 \n", "four_up_return_day14_return_count 0.054255 \n", "four_up_return_day1_return_count 0.024934 \n", "four_up_return_day3_return_count 0.033559 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.018049 \n", "day3ctr 0.003958 \n", "day7ctr 0.007342 \n", "\n", " non_zero_mean_0.75 \\\n", "feature_name \n", "stage_four_retrn 0.010414 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.004514 \n", "stage_two_retrn 0.002557 \n", "stage_three_retrn 0.003311 \n", "stage_two_return_added -0.003437 \n", "four_up_return_day1_view_day1_return_count 0.000774 \n", "four_up_return_div_three_return_day1 0.016906 \n", "day1playcount 0.022199 \n", "todyviewcount_rank 0.034629 \n", "usercategory8 0.083957 \n", "usercategory49 0.065647 \n", "day3viewcount 0.078234 \n", "day7viewcount 0.066762 \n", "usercategory12 0.030142 \n", "usercategory6 0.013628 \n", "usercategory45 0.040219 \n", "usercategory10 0.003313 \n", "usercategory85 0.050542 \n", "usercategory1 0.000484 \n", "four_up_return_day3_view_day3_return_count 0.030331 \n", "usercategory7 0.041463 \n", "all_return_day1_return_count 0.012683 \n", "usercategory9 0.058713 \n", "three_return_day1_return_count 0.013850 \n", "four_up_return_day7_return_count 0.054221 \n", "all_return_day14_return_count 0.026809 \n", "todyviewcount 0.092298 \n", "usercategory11 0.035500 \n", "usercategory4 0.042675 \n", "four_up_return_day14_return_count 0.062747 \n", "four_up_return_day1_return_count 0.041597 \n", "four_up_return_day3_return_count 0.051668 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.014849 \n", "day3ctr 0.001072 \n", "day7ctr 0.003971 \n", "\n", " non_zero_mean_1 \\\n", "feature_name \n", "stage_four_retrn 0.057912 \n", "videocategory2 0.000000 \n", "videocategory10 0.000000 \n", "videocategory6 0.000000 \n", "videocategory8 0.000000 \n", "videocategory12 0.000000 \n", "videocategory45 0.000000 \n", "stage_one_retrn 0.012775 \n", "stage_two_retrn 0.024518 \n", "stage_three_retrn 0.040968 \n", "stage_two_return_added -0.003670 \n", "four_up_return_day1_view_day1_return_count 0.025033 \n", "four_up_return_div_three_return_day1 0.017316 \n", "day1playcount 0.022780 \n", "todyviewcount_rank 0.058987 \n", "usercategory8 0.042837 \n", "usercategory49 0.032021 \n", "day3viewcount 0.052805 \n", "day7viewcount 0.050658 \n", "usercategory12 0.014068 \n", "usercategory6 0.005199 \n", "usercategory45 0.033315 \n", "usercategory10 0.008161 \n", "usercategory85 0.005018 \n", "usercategory1 0.005178 \n", "four_up_return_day3_view_day3_return_count 0.017286 \n", "usercategory7 0.032458 \n", "all_return_day1_return_count 0.026116 \n", "usercategory9 0.028566 \n", "three_return_day1_return_count 0.025082 \n", "four_up_return_day7_return_count 0.036746 \n", "all_return_day14_return_count 0.028682 \n", "todyviewcount 0.054649 \n", "usercategory11 0.008586 \n", "usercategory4 0.023515 \n", "four_up_return_day14_return_count 0.031379 \n", "four_up_return_day1_return_count 0.033956 \n", "four_up_return_day3_return_count 0.030638 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.002818 \n", "day3ctr 0.011206 \n", "day7ctr 0.007188 \n", "\n", " non_zero_var_0.25 \\\n", "feature_name \n", "stage_four_retrn 0.010849 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.000702 \n", "stage_two_retrn 0.018237 \n", "stage_three_retrn 0.000559 \n", "stage_two_return_added 0.022244 \n", "four_up_return_day1_view_day1_return_count 0.008815 \n", "four_up_return_div_three_return_day1 0.036767 \n", "day1playcount 0.012271 \n", "todyviewcount_rank 0.067377 \n", "usercategory8 0.379763 \n", "usercategory49 0.576893 \n", "day3viewcount 0.557694 \n", "day7viewcount 0.883487 \n", "usercategory12 0.130214 \n", "usercategory6 0.139570 \n", "usercategory45 0.427017 \n", "usercategory10 0.166045 \n", "usercategory85 0.169806 \n", "usercategory1 0.114492 \n", "four_up_return_day3_view_day3_return_count 0.102537 \n", "usercategory7 0.122846 \n", "all_return_day1_return_count 0.124121 \n", "usercategory9 0.209519 \n", "three_return_day1_return_count 0.645306 \n", "four_up_return_day7_return_count 0.143251 \n", "all_return_day14_return_count 0.134059 \n", "todyviewcount NaN \n", "usercategory11 0.009885 \n", "usercategory4 0.030476 \n", "four_up_return_day14_return_count 0.097179 \n", "four_up_return_day1_return_count 0.024356 \n", "four_up_return_day3_return_count 0.093090 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.019762 \n", "day3ctr 0.000536 \n", "day7ctr 0.001180 \n", "\n", " non_zero_var_0.5 \\\n", "feature_name \n", "stage_four_retrn 0.018093 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.008503 \n", "stage_two_retrn 0.000353 \n", "stage_three_retrn 0.006996 \n", "stage_two_return_added 0.021602 \n", "four_up_return_day1_view_day1_return_count 0.009560 \n", "four_up_return_div_three_return_day1 0.041840 \n", "day1playcount 0.091721 \n", "todyviewcount_rank 0.067377 \n", "usercategory8 0.209333 \n", "usercategory49 0.159956 \n", "day3viewcount 0.032570 \n", "day7viewcount 0.074300 \n", "usercategory12 0.152885 \n", "usercategory6 0.012217 \n", "usercategory45 0.131324 \n", "usercategory10 0.020356 \n", "usercategory85 0.117835 \n", "usercategory1 0.013757 \n", "four_up_return_day3_view_day3_return_count 0.060346 \n", "usercategory7 0.127237 \n", "all_return_day1_return_count 0.017365 \n", "usercategory9 0.183533 \n", "three_return_day1_return_count 0.027203 \n", "four_up_return_day7_return_count 0.089129 \n", "all_return_day14_return_count 0.059231 \n", "todyviewcount 0.234774 \n", "usercategory11 0.136667 \n", "usercategory4 0.106830 \n", "four_up_return_day14_return_count 0.121594 \n", "four_up_return_day1_return_count 0.085318 \n", "four_up_return_day3_return_count 0.079053 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.032172 \n", "day3ctr 0.011389 \n", "day7ctr 0.013882 \n", "\n", " non_zero_var_0.75 \\\n", "feature_name \n", "stage_four_retrn 0.020078 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.013407 \n", "stage_two_retrn 0.005393 \n", "stage_three_retrn 0.006591 \n", "stage_two_return_added 0.021383 \n", "four_up_return_day1_view_day1_return_count 0.006945 \n", "four_up_return_div_three_return_day1 0.032526 \n", "day1playcount 0.041539 \n", "todyviewcount_rank 0.069366 \n", "usercategory8 0.100997 \n", "usercategory49 0.066917 \n", "day3viewcount 0.033151 \n", "day7viewcount 0.059949 \n", "usercategory12 0.010870 \n", "usercategory6 0.022983 \n", "usercategory45 0.043284 \n", "usercategory10 0.004968 \n", "usercategory85 0.111731 \n", "usercategory1 0.018887 \n", "four_up_return_day3_view_day3_return_count 0.061611 \n", "usercategory7 0.060318 \n", "all_return_day1_return_count 0.033697 \n", "usercategory9 0.072907 \n", "three_return_day1_return_count 0.036757 \n", "four_up_return_day7_return_count 0.120658 \n", "all_return_day14_return_count 0.056375 \n", "todyviewcount 0.215959 \n", "usercategory11 0.051559 \n", "usercategory4 0.073398 \n", "four_up_return_day14_return_count 0.132839 \n", "four_up_return_day1_return_count 0.100219 \n", "four_up_return_day3_return_count 0.118647 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.025368 \n", "day3ctr 0.019178 \n", "day7ctr 0.019550 \n", "\n", " non_zero_var_1 \n", "feature_name \n", "stage_four_retrn 1.277436 \n", "videocategory2 NaN \n", "videocategory10 NaN \n", "videocategory6 NaN \n", "videocategory8 NaN \n", "videocategory12 NaN \n", "videocategory45 NaN \n", "stage_one_retrn 0.151157 \n", "stage_two_retrn 0.436318 \n", "stage_three_retrn 0.912625 \n", "stage_two_return_added 0.058964 \n", "four_up_return_day1_view_day1_return_count 0.203595 \n", "four_up_return_div_three_return_day1 0.466700 \n", "day1playcount 0.062169 \n", "todyviewcount_rank 0.172902 \n", "usercategory8 0.040812 \n", "usercategory49 0.033942 \n", "day3viewcount 0.069517 \n", "day7viewcount 0.069719 \n", "usercategory12 0.040111 \n", "usercategory6 0.007871 \n", "usercategory45 0.045161 \n", "usercategory10 0.010546 \n", "usercategory85 0.007461 \n", "usercategory1 0.005524 \n", "four_up_return_day3_view_day3_return_count 0.003348 \n", "usercategory7 0.048708 \n", "all_return_day1_return_count 0.027357 \n", "usercategory9 0.043999 \n", "three_return_day1_return_count 0.034166 \n", "four_up_return_day7_return_count 0.029382 \n", "all_return_day14_return_count 0.037552 \n", "todyviewcount 0.076444 \n", "usercategory11 0.005929 \n", "usercategory4 0.034067 \n", "four_up_return_day14_return_count 0.017569 \n", "four_up_return_day1_return_count 0.017424 \n", "four_up_return_day3_return_count 0.017275 \n", "four_up_return_day3_view_day3_return_div_three_d3 0.104871 \n", "day3ctr 0.114604 \n", "day7ctr 0.102625 \n" ] } ], "source": [ "file_20220221 = './data/train_data_monitor_20220221.csv'\n", "file_20220220 = './data/train_data_monitor_20220220.csv'\n", "offset_feature_df = cal_diff2(yesterday_file=file_20220220, today_file=file_20220221, feature_top=-1)\n", "print(offset_feature_df)" ] }, { "cell_type": "code", "execution_count": 90, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 var_1 \\\n", "feature_name \n", "usercategory12 0.389068 0.149047 0.202083 \n", "usercategory6 0.349599 1.848714 0.512809 \n", "usercategory45 0.248777 0.286826 0.562451 \n", "usercategory10 0.400805 0.509582 0.166816 \n", "usercategory11 0.385189 0.275260 0.227190 \n", "... ... ... ... \n", "three_return_day14_view_day14_return_count 0.018287 0.060189 0.040123 \n", "day14sharecount 0.021091 0.076526 0.012636 \n", "day7returncount 0.017678 0.054290 0.088003 \n", "day1ctr 0.020976 0.013847 0.017291 \n", "day7sharecount 0.017844 0.057830 0.053031 \n", "\n", " non_zero_mean_0.25 \\\n", "feature_name \n", "usercategory12 0.158467 \n", "usercategory6 0.124405 \n", "usercategory45 0.000000 \n", "usercategory10 0.000000 \n", "usercategory11 0.302892 \n", "... ... \n", "three_return_day14_view_day14_return_count 0.040742 \n", "day14sharecount 0.000000 \n", "day7returncount 0.010657 \n", "day1ctr 0.045589 \n", "day7sharecount 0.000000 \n", "\n", " non_zero_mean_0.5 \\\n", "feature_name \n", "usercategory12 0.467306 \n", "usercategory6 1.543319 \n", "usercategory45 0.349660 \n", "usercategory10 0.742160 \n", "usercategory11 0.840701 \n", "... ... \n", "three_return_day14_view_day14_return_count 0.044874 \n", "day14sharecount 0.025322 \n", "day7returncount 0.061121 \n", "day1ctr 0.038261 \n", "day7sharecount 0.020494 \n", "\n", " non_zero_mean_0.75 \\\n", "feature_name \n", "usercategory12 0.491538 \n", "usercategory6 0.587295 \n", "usercategory45 0.356017 \n", "usercategory10 0.467726 \n", "usercategory11 0.403284 \n", "... ... \n", "three_return_day14_view_day14_return_count 0.093961 \n", "day14sharecount 0.079782 \n", "day7returncount 0.099012 \n", "day1ctr 0.045860 \n", "day7sharecount 0.081783 \n", "\n", " non_zero_mean_1 \\\n", "feature_name \n", "usercategory12 0.092711 \n", "usercategory6 0.235541 \n", "usercategory45 0.422256 \n", "usercategory10 0.184633 \n", "usercategory11 0.013129 \n", "... ... \n", "three_return_day14_view_day14_return_count 0.002698 \n", "day14sharecount 0.040768 \n", "day7returncount 0.002931 \n", "day1ctr 0.042146 \n", "day7sharecount 0.026941 \n", "\n", " non_zero_var_0.25 \\\n", "feature_name \n", "usercategory12 0.279697 \n", "usercategory6 0.107641 \n", "usercategory45 NaN \n", "usercategory10 NaN \n", "usercategory11 0.488379 \n", "... ... \n", "three_return_day14_view_day14_return_count 0.241747 \n", "day14sharecount NaN \n", "day7returncount 0.032405 \n", "day1ctr 0.060760 \n", "day7sharecount NaN \n", "\n", " non_zero_var_0.5 \\\n", "feature_name \n", "usercategory12 4.294396 \n", "usercategory6 24.292716 \n", "usercategory45 0.866082 \n", "usercategory10 14.314274 \n", "usercategory11 5.005170 \n", "... ... \n", "three_return_day14_view_day14_return_count 0.077078 \n", "day14sharecount 0.139806 \n", "day7returncount 0.171901 \n", "day1ctr 0.049868 \n", "day7sharecount 0.079958 \n", "\n", " non_zero_var_0.75 non_zero_var_1 \n", "feature_name \n", "usercategory12 1.309163 0.363100 \n", "usercategory6 0.429224 0.892191 \n", "usercategory45 0.599201 0.639164 \n", "usercategory10 0.635872 0.330137 \n", "usercategory11 0.289225 0.388015 \n", "... ... ... \n", "three_return_day14_view_day14_return_count 0.224951 0.104362 \n", "day14sharecount 0.243155 0.026449 \n", "day7returncount 0.233276 0.154486 \n", "day1ctr 0.128236 0.088512 \n", "day7sharecount 0.252405 0.088462 \n", "\n", "[120 rows x 11 columns]\n" ] } ], "source": [ "file_20220221 = './data/predict_data_monitor_20220221.csv'\n", "file_20220220 = './data/predict_data_monitor_20220220.csv'\n", "offset_feature_df = cal_diff2(yesterday_file=file_20220220, today_file=file_20220221, feature_top=-1)\n", "print(offset_feature_df)" ] }, { "cell_type": "code", "execution_count": 92, "metadata": { "scrolled": true, "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " 0_rate mean_1 var_1 non_zero_mean_0.25 \\\n", "feature_name \n", "usercategory12 0.161520 0.161136 0.323249 0.283967 \n", "usercategory6 0.433782 0.839190 0.982490 0.495206 \n", "usercategory45 0.103797 0.544485 0.810778 0.149898 \n", "usercategory10 0.484662 0.879278 0.981654 0.454628 \n", "usercategory11 0.112797 0.343456 0.760425 0.553168 \n", "usercategory85 0.791914 0.491879 0.938848 0.185439 \n", "day3ctr 0.509684 0.451490 0.127555 0.076134 \n", "day7sov 0.101638 0.087710 0.572643 0.008470 \n", "usercategory1 0.603805 0.884418 0.973853 0.585021 \n", "day14sov 0.112074 0.293491 0.603897 0.010271 \n", "day14ctr 0.550332 0.539432 0.043038 0.078439 \n", "day7ctr 0.520152 0.399568 0.188191 0.082829 \n", "day7soc 0.102343 0.082909 0.513177 0.071056 \n", "day14soc 0.112187 0.230600 0.921450 0.071260 \n", "usercategory49 0.111148 0.073147 0.163257 1.579664 \n", "usercategory7 0.598655 0.607635 0.787458 0.270320 \n", "usercategory5 0.316902 0.705185 0.914827 0.403641 \n", "day1viewcount 0.553721 0.186579 0.437704 0.598604 \n", "usercategory9 0.944148 0.448628 0.573008 1.346846 \n", "day3playcount 0.509665 0.345503 0.955884 0.236031 \n", "\n", " non_zero_mean_0.5 non_zero_mean_0.75 non_zero_mean_1 \\\n", "feature_name \n", "usercategory12 0.026433 0.126495 0.214523 \n", "usercategory6 0.036778 0.372643 0.772966 \n", "usercategory45 0.365566 0.387731 0.519922 \n", "usercategory10 0.597940 0.768367 0.852184 \n", "usercategory11 0.417395 0.003474 0.321622 \n", "usercategory85 0.295629 0.822159 0.373553 \n", "day3ctr 0.071096 0.064645 0.035975 \n", "day7sov 0.024097 0.003913 0.142419 \n", "usercategory1 0.696831 0.782710 0.868027 \n", "day14sov 0.025886 0.000910 0.012570 \n", "day14ctr 0.071455 0.065724 0.055538 \n", "day7ctr 0.070946 0.060532 0.022918 \n", "day7soc 0.073734 0.078272 0.148164 \n", "day14soc 0.068903 0.070241 0.061147 \n", "usercategory49 0.736347 0.247594 0.054282 \n", "usercategory7 0.280280 0.473937 0.571051 \n", "usercategory5 0.500944 0.500485 0.717739 \n", "day1viewcount 0.673151 0.428038 0.173003 \n", "usercategory9 0.113509 0.278851 0.383804 \n", "day3playcount 0.096735 0.020572 0.039642 \n", "\n", " non_zero_var_0.25 non_zero_var_0.5 non_zero_var_0.75 \\\n", "feature_name \n", "usercategory12 0.559054 0.534133 0.396978 \n", "usercategory6 0.840472 0.554074 0.794261 \n", "usercategory45 1.000000 0.810464 0.671310 \n", "usercategory10 1.000000 0.888928 0.983323 \n", "usercategory11 0.715267 0.931129 0.454126 \n", "usercategory85 0.533149 2.713827 2.856045 \n", "day3ctr 0.054567 0.102053 0.083922 \n", "day7sov 0.239618 0.128438 0.102590 \n", "usercategory1 0.932713 0.926920 0.975288 \n", "day14sov 0.253165 0.133330 0.077624 \n", "day14ctr 0.075536 0.097081 0.085262 \n", "day7ctr 0.066286 0.069821 0.040820 \n", "day7soc 0.069266 0.150046 0.175688 \n", "day14soc 0.060550 0.122720 0.139181 \n", "usercategory49 6.590469 1.347639 0.071248 \n", "usercategory7 0.214642 0.480430 0.842149 \n", "usercategory5 0.767272 0.738645 0.781385 \n", "day1viewcount 0.981925 0.888958 0.325608 \n", "usercategory9 2.439561 0.366690 0.744477 \n", "day3playcount 0.086158 0.178202 0.424323 \n", "\n", " non_zero_var_1 \n", "feature_name \n", "usercategory12 0.360183 \n", "usercategory6 0.979666 \n", "usercategory45 0.802399 \n", "usercategory10 0.978218 \n", "usercategory11 0.766784 \n", "usercategory85 0.945581 \n", "day3ctr 0.271520 \n", "day7sov 0.670824 \n", "usercategory1 0.970089 \n", "day14sov 0.241328 \n", "day14ctr 0.026570 \n", "day7ctr 0.305590 \n", "day7soc 0.620550 \n", "day14soc 0.487286 \n", "usercategory49 0.150713 \n", "usercategory7 0.768257 \n", "usercategory5 0.918050 \n", "day1viewcount 0.005644 \n", "usercategory9 0.524942 \n", "day3playcount 0.398926 \n" ] } ], "source": [ "file_20220221 = './data/predict_data_monitor_20220221.csv'\n", "file_20220220 = './data/train_data_monitor_20220221.csv'\n", "offset_feature_df = cal_diff2(yesterday_file=file_20220220, today_file=file_20220221, feature_top=-1)\n", "print(offset_feature_df.head(20))" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\"Extra\":null,\"StatusCode\":0,\"StatusMessage\":\"success\"}\n" ] } ], "source": [ "msg_text = '\\ndate: 20220221 \\ntag: train \\n{}'.format(offset_feature_df)\n", "send_msg_to_feishu(msg_text)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" } }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }