rov-offline/data_monitor.py

# 对训练数据的分布进行监控
import numpy as np
import pandas as pd
import datetime

from config import set_config
from rov_train import process_data, process_predict_data

config_, env = set_config()


def get_feature_distribution(feature_name, feature_data):
    statistical_results = {'feature_name': feature_name}
    feature_data = np.array(feature_data)
    feature_data_sorted = sorted(feature_data)
    length = len(feature_data_sorted)
    count_0 = len([item for item in feature_data_sorted if item == 0])
    print('data_count = {}, count_0 = {}, rate_0 = {}'.format(length, count_0, count_0/length))
    statistical_results['data_count'] = length
    statistical_results['0_count'] = count_0
    statistical_results['0_rate'] = count_0/length

    # 整体数据分布
    for percentile in [0.25, 0.5, 0.75, 1]:
        data_count = int(length * percentile)
        data = feature_data_sorted[:data_count + 1]
        data_mean = np.mean(data)
        data_var = np.var(data)
        data_std = np.std(data)
        # print('percentile = {}, data_count = {}, mean = {}, var = {}, std = {}'.format(
        #     percentile, data_count, data_mean, data_var, data_std))
        statistical_results['mean_{}'.format(percentile)] = data_mean
        statistical_results['var_{}'.format(percentile)] = data_var
        statistical_results['std_{}'.format(percentile)] = data_std

    # 非零数据分布
    data_non_zero = [item for item in feature_data_sorted if item != 0]
    for percentile in [0.25, 0.5, 0.75, 1]:
        data_count = int(len(data_non_zero) * percentile)
        data = data_non_zero[:data_count + 1]
        data_mean = np.mean(data)
        data_var = np.var(data)
        dat_std = np.std(data)
        # print('percentile = {}, data_count = {}, mean = {}, var = {}, std = {}'.format(
        #     percentile, data_count, data_mean, data_var, dat_std))
        statistical_results['non_zero_mean_{}'.format(percentile)] = data_mean
        statistical_results['non_zero_var_{}'.format(percentile)] = data_var
        statistical_results['non_zero_std_{}'.format(percentile)] = data_std

    return statistical_results


def all_feature_distribution(data, file):
    res = []
    columns = [
        'feature_name', 'data_count', '0_count', '0_rate',
        'mean_0.25', 'mean_0.5', 'mean_0.75', 'mean_1',
        'var_0.25', 'var_0.5', 'var_0.75', 'var_1',
        'std_0.25', 'std_0.5', 'std_0.75', 'std_1',
        'non_zero_mean_0.25', 'non_zero_mean_0.5', 'non_zero_mean_0.75', 'non_zero_mean_1',
        'non_zero_var_0.25', 'non_zero_var_0.5', 'non_zero_var_0.75', 'non_zero_var_1',
        'non_zero_std_0.25', 'non_zero_std_0.5', 'non_zero_std_0.75', 'non_zero_std_1'
    ]
    feature_importance = pd.read_csv('data/model_feature_importance.csv')
    feature_name_list = list(feature_importance['feature'])
    for feature_name in feature_name_list:
        print(feature_name)
        feature_data = data[feature_name]
        statistical_results = get_feature_distribution(feature_name=feature_name, feature_data=feature_data)
        res.append(statistical_results)
    df = pd.DataFrame(res, columns=columns)
    df.to_csv(file)


def main():
    now_date = datetime.datetime.strftime(datetime.datetime.today(), '%Y%m%d')
    # now_date = '20220119'

    # 训练数据
    print('train data monitor...')
    train_data_file = 'data/train_data_monitor_{}.csv'.format(now_date)
    train_filename = config_.TRAIN_DATA_FILENAME
    train_x, train_y, videos, fea = process_data(filename=train_filename)
    all_feature_distribution(train_x, file=train_data_file)

    # 预测数据
    print('predict data monitor...')
    predict_data_file = 'data/predict_data_monitor_{}.csv'.format(now_date)
    predict_filename = config_.PREDICT_DATA_FILENAME
    predict_x, video_ids = process_predict_data(filename=predict_filename)
    all_feature_distribution(predict_x, file=predict_data_file)


if __name__ == '__main__':
    main()