#!/usr/bin/env python
# coding=utf-8
"""
Step3b: 分析整体预估偏差（所有头部视频）
"""
import pandas as pd
from pathlib import Path
import glob

# 读取并合并数据
data_dir = Path(__file__).parent / "output" / "step3b_整体偏差"
all_files = glob.glob(str(data_dir / "*.csv"))
df = pd.concat([pd.read_csv(f) for f in all_files], ignore_index=True)

print("=" * 70)
print("Step3b: 整体预估偏差分析")
print("=" * 70)

# 1. 数据概览
print("\n【1. 数据概览】")
print(f"样本数: {len(df)} 条")
print(f"视频数: {df['vid'].nunique()}")
print(f"日期范围: {df['dt'].min()} ~ {df['dt'].max()}")

# 2. 过滤有效数据（real_ros 不为空）
df_valid = df[df['real_ros'].notna() & (df['real_ros'] > 0)].copy()
print(f"有效样本: {len(df_valid)} 条（real_ros > 0）")

# 3. 计算偏差
df_valid['str_bias'] = df_valid['pred_str'] - df_valid['real_str']
df_valid['ros_bias'] = df_valid['pred_ros'] - df_valid['real_ros']
df_valid['vor_bias'] = df_valid['pred_vor'] - df_valid['real_vor']

df_valid['str_bias_pct'] = (df_valid['pred_str'] - df_valid['real_str']) / df_valid['real_str'] * 100
df_valid['ros_bias_pct'] = (df_valid['pred_ros'] - df_valid['real_ros']) / df_valid['real_ros'] * 100
df_valid['vor_bias_pct'] = (df_valid['pred_vor'] - df_valid['real_vor']) / df_valid['real_vor'] * 100

# 4. 整体偏差统计
print("\n【2. 整体偏差统计】")
print(f"STR: 预估均值={df_valid['pred_str'].mean():.6f}, 真实均值={df_valid['real_str'].mean():.6f}, 偏差={df_valid['str_bias_pct'].mean():+.1f}%")
print(f"ROS: 预估均值={df_valid['pred_ros'].mean():.4f}, 真实均值={df_valid['real_ros'].mean():.4f}, 偏差={df_valid['ros_bias_pct'].mean():+.1f}%")
print(f"VOR: 预估均值={df_valid['pred_vor'].mean():.4f}, 真实均值={df_valid['real_vor'].mean():.4f}, 偏差={df_valid['vor_bias_pct'].mean():+.1f}%")

# 5. 偏差分布
print("\n【3. 偏差分布】")
for col, name in [('str_bias_pct', 'STR'), ('ros_bias_pct', 'ROS'), ('vor_bias_pct', 'VOR')]:
    high = (df_valid[col] > 0).sum()
    low = (df_valid[col] < 0).sum()
    print(f"{name}: 偏高{high}个({high/len(df_valid)*100:.1f}%), 偏低{low}个({low/len(df_valid)*100:.1f}%)")

# 6. 按曝光量分组看偏差
print("\n【4. 按曝光量分组看偏差】")
df_valid['exp_group'] = pd.qcut(df_valid['sample_cnt'], q=4, labels=['低曝光', '中低曝光', '中高曝光', '高曝光'])
group_stats = df_valid.groupby('exp_group', observed=True).agg({
    'str_bias_pct': 'mean',
    'ros_bias_pct': 'mean',
    'vor_bias_pct': 'mean',
    'sample_cnt': ['mean', 'count']
}).round(2)
group_stats.columns = ['STR偏差%', 'ROS偏差%', 'VOR偏差%', '平均样本', '视频数']
print(group_stats.to_string())

# 7. 结论
print("\n" + "=" * 70)
print("【Step3b 结论】")
print("=" * 70)

str_bias = df_valid['str_bias_pct'].mean()
ros_bias = df_valid['ros_bias_pct'].mean()
vor_bias = df_valid['vor_bias_pct'].mean()

print(f"整体偏差情况:")
print(f"  STR: {str_bias:+.1f}% {'(偏高)' if str_bias > 10 else '(偏低)' if str_bias < -10 else '(正常)'}")
print(f"  ROS: {ros_bias:+.1f}% {'(偏高)' if ros_bias > 10 else '(偏低)' if ros_bias < -10 else '(正常)'}")
print(f"  VOR: {vor_bias:+.1f}% {'(偏高)' if vor_bias > 10 else '(偏低)' if vor_bias < -10 else '(正常)'}")

# 保存结果
output_dir = Path(__file__).parent / "output"
df_valid.to_csv(output_dir / "step3b_整体偏差_合并.csv", index=False)

with open(output_dir / "step3b_结论.txt", 'w', encoding='utf-8') as f:
    f.write("=" * 70 + "\n")
    f.write("Step3b: 整体预估偏差分析\n")
    f.write("=" * 70 + "\n\n")
    f.write(f"样本: {len(df_valid)} 条, {df_valid['vid'].nunique()} 个视频\n\n")
    f.write("【整体偏差】\n")
    f.write(f"STR: {str_bias:+.1f}%\n")
    f.write(f"ROS: {ros_bias:+.1f}%\n")
    f.write(f"VOR: {vor_bias:+.1f}%\n")

print(f"\n结果已保存到 output/ 目录")
