#!/usr/bin/env python
# coding=utf-8
"""
V2 深入分析：全面、严谨的分析
"""
import pandas as pd
import numpy as np
from pathlib import Path
import glob
from datetime import datetime

output_dir = Path(__file__).parent / "output"

# 读取30天数据
data_dir = output_dir / "v2_step2_统一口径"
all_files = sorted(glob.glob(str(data_dir / "*.csv")))
dfs = []
for f in all_files:
    df_tmp = pd.read_csv(f)
    df_tmp['dt'] = int(Path(f).stem)
    dfs.append(df_tmp)
df = pd.concat(dfs, ignore_index=True)

print("=" * 80)
print("V2 深入分析：30天数据全面分析")
print("=" * 80)

print(f"\n数据概览:")
print(f"  日期范围: {df['dt'].min()} ~ {df['dt'].max()}")
print(f"  天数: {df['dt'].nunique()}")
print(f"  总视频数: {len(df)}")
print(f"  去重视频数: {df['vid'].nunique()}")

# =============================================================================
# Part 1: 整体指标
# =============================================================================
print("\n" + "=" * 80)
print("Part 1: 整体指标")
print("=" * 80)

# COPC
ros_copc = df['real_ros'].sum() / df['pred_ros'].sum()
str_copc = df['real_str'].sum() / df['pred_str'].sum()

print(f"\n【1.1 COPC 整体】")
print(f"  ROS COPC = {ros_copc:.4f} ({'预估偏低' if ros_copc > 1 else '预估偏高'})")
print(f"  STR COPC = {str_copc:.4f} ({'预估偏低' if str_copc > 1 else '预估偏高'})")

# 偏差分布
print(f"\n【1.2 偏差分布】")
print(f"  ROS偏差: 均值={df['ros_bias_pct'].mean():+.1f}%, 中位数={df['ros_bias_pct'].median():+.1f}%, std={df['ros_bias_pct'].std():.1f}%")
print(f"  STR偏差: 均值={df['str_bias_pct'].mean():+.1f}%, 中位数={df['str_bias_pct'].median():+.1f}%, std={df['str_bias_pct'].std():.1f}%")

# 偏差分位数
print(f"\n【1.3 ROS偏差分位数】")
for p in [0.1, 0.25, 0.5, 0.75, 0.9]:
    val = df['ros_bias_pct'].quantile(p)
    print(f"  P{int(p*100)}: {val:+.1f}%")

# =============================================================================
# Part 2: 分段COPC（回归均值分析）
# =============================================================================
print("\n" + "=" * 80)
print("Part 2: 分段COPC分析（验证回归均值问题）")
print("=" * 80)

print(f"\n【2.1 按真实ROS分段】")
df['real_ros_bin'] = pd.cut(df['real_ros'], bins=[0, 1, 2, 3, 4, 6, 10, 100],
                           labels=['0-1', '1-2', '2-3', '3-4', '4-6', '6-10', '>10'])
ros_segment = df.groupby('real_ros_bin', observed=True).agg({
    'pred_ros': 'mean',
    'real_ros': 'mean',
    'ros_bias_pct': ['mean', 'median', 'count']
}).round(2)
ros_segment.columns = ['预估ROS', '真实ROS', '偏差均值%', '偏差中位数%', '样本数']
ros_segment['COPC'] = (ros_segment['真实ROS'] / ros_segment['预估ROS']).round(3)
print(ros_segment.to_string())

print(f"\n【2.2 按真实STR分段】")
df['real_str_bin'] = pd.cut(df['real_str'], bins=[0, 0.005, 0.01, 0.02, 0.05, 1],
                           labels=['<0.5%', '0.5-1%', '1-2%', '2-5%', '>5%'])
str_segment = df.groupby('real_str_bin', observed=True).agg({
    'pred_str': 'mean',
    'real_str': 'mean',
    'str_bias_pct': ['mean', 'count']
}).round(4)
str_segment.columns = ['预估STR', '真实STR', '偏差均值%', '样本数']
print(str_segment.to_string())

# =============================================================================
# Part 3: 排序效果分析
# =============================================================================
print("\n" + "=" * 80)
print("Part 3: 排序效果分析")
print("=" * 80)

# 按天计算排序相关性
print(f"\n【3.1 每日排序相关性】")
daily_corr = []
for dt in sorted(df['dt'].unique()):
    df_day = df[df['dt'] == dt].copy()
    if len(df_day) > 10:
        df_day['pred_rank'] = df_day['pred_score'].rank(ascending=False)
        df_day['real_rank'] = df_day['real_score'].rank(ascending=False)
        corr = df_day['pred_rank'].corr(df_day['real_rank'])
        daily_corr.append({'dt': dt, 'corr': corr, 'n': len(df_day)})

corr_df = pd.DataFrame(daily_corr)
print(f"  平均相关系数: {corr_df['corr'].mean():.4f}")
print(f"  最低: {corr_df['corr'].min():.4f} ({corr_df.loc[corr_df['corr'].idxmin(), 'dt']})")
print(f"  最高: {corr_df['corr'].max():.4f} ({corr_df.loc[corr_df['corr'].idxmax(), 'dt']})")
print(f"  标准差: {corr_df['corr'].std():.4f}")

# 排序失准比例
print(f"\n【3.2 排序失准分析】")
problem_stats = []
for dt in sorted(df['dt'].unique()):
    df_day = df[df['dt'] == dt].copy()
    if len(df_day) > 10:
        df_day['pred_rank'] = df_day['pred_score'].rank(ascending=False)
        df_day['real_rank'] = df_day['real_score'].rank(ascending=False)
        n = len(df_day)
        top25 = n * 0.25
        bottom50 = n * 0.5
        problem_cnt = ((df_day['pred_rank'] <= top25) & (df_day['real_rank'] > bottom50)).sum()
        problem_stats.append({'dt': dt, 'problem_pct': problem_cnt / len(df_day) * 100})

problem_df = pd.DataFrame(problem_stats)
print(f"  平均问题比例: {problem_df['problem_pct'].mean():.1f}%")
print(f"  最低: {problem_df['problem_pct'].min():.1f}%")
print(f"  最高: {problem_df['problem_pct'].max():.1f}%")

# =============================================================================
# Part 4: 时间趋势
# =============================================================================
print("\n" + "=" * 80)
print("Part 4: 时间趋势分析")
print("=" * 80)

# 按天的COPC趋势
print(f"\n【4.1 每日COPC趋势】")
daily_stats = df.groupby('dt').agg({
    'pred_ros': 'sum',
    'real_ros': 'sum',
    'ros_bias_pct': 'mean',
    'vid': 'count'
}).round(4)
daily_stats['ros_copc'] = daily_stats['real_ros'] / daily_stats['pred_ros']
daily_stats = daily_stats.round(4)

print(f"{'日期':<12} {'样本数':>8} {'ROS COPC':>10} {'偏差均值%':>12}")
print("-" * 45)
for dt, row in daily_stats.iterrows():
    print(f"{dt:<12} {int(row['vid']):>8} {row['ros_copc']:>10.3f} {row['ros_bias_pct']:>+11.1f}%")

# 周度汇总
print(f"\n【4.2 周度汇总】")
df['week'] = pd.to_datetime(df['dt'].astype(str)).dt.isocalendar().week
weekly = df.groupby('week').agg({
    'pred_ros': 'sum',
    'real_ros': 'sum',
    'ros_bias_pct': 'mean',
    'vid': 'count'
})
weekly['ros_copc'] = (weekly['real_ros'] / weekly['pred_ros']).round(3)
print(weekly[['vid', 'ros_copc', 'ros_bias_pct']].round(2).to_string())

# =============================================================================
# Part 5: 问题视频深入分析
# =============================================================================
print("\n" + "=" * 80)
print("Part 5: 问题视频深入分析")
print("=" * 80)

# 定义问题视频（每天独立计算）
df['pred_rank_daily'] = df.groupby('dt')['pred_score'].rank(ascending=False, pct=True)
df['real_rank_daily'] = df.groupby('dt')['real_score'].rank(ascending=False, pct=True)
df['is_problem'] = (df['pred_rank_daily'] <= 0.25) & (df['real_rank_daily'] > 0.5)

problem = df[df['is_problem']]
normal = df[~df['is_problem']]

print(f"\n【5.1 问题视频统计】")
print(f"  问题视频: {len(problem)} ({len(problem)/len(df)*100:.1f}%)")
print(f"  正常视频: {len(normal)} ({len(normal)/len(df)*100:.1f}%)")

print(f"\n【5.2 问题视频 vs 正常视频对比】")
metrics = ['pred_ros', 'real_ros', 'ros_bias_pct', 'pred_str', 'real_str', 'str_bias_pct', 'total_exp']
comparison = pd.DataFrame({
    '问题视频': problem[metrics].mean(),
    '正常视频': normal[metrics].mean(),
})
comparison['差异'] = comparison['问题视频'] - comparison['正常视频']
print(comparison.round(4).to_string())

print(f"\n【5.3 问题视频的真实ROS分布】")
print("问题视频:")
print(f"  ROS < 2: {(problem['real_ros'] < 2).sum()} ({(problem['real_ros'] < 2).mean()*100:.1f}%)")
print(f"  ROS 2-4: {((problem['real_ros'] >= 2) & (problem['real_ros'] < 4)).sum()} ({((problem['real_ros'] >= 2) & (problem['real_ros'] < 4)).mean()*100:.1f}%)")
print(f"  ROS > 4: {(problem['real_ros'] >= 4).sum()} ({(problem['real_ros'] >= 4).mean()*100:.1f}%)")
print("\n正常视频:")
print(f"  ROS < 2: {(normal['real_ros'] < 2).sum()} ({(normal['real_ros'] < 2).mean()*100:.1f}%)")
print(f"  ROS 2-4: {((normal['real_ros'] >= 2) & (normal['real_ros'] < 4)).sum()} ({((normal['real_ros'] >= 2) & (normal['real_ros'] < 4)).mean()*100:.1f}%)")
print(f"  ROS > 4: {(normal['real_ros'] >= 4).sum()} ({(normal['real_ros'] >= 4).mean()*100:.1f}%)")

# =============================================================================
# Part 6: 高频问题视频分析
# =============================================================================
print("\n" + "=" * 80)
print("Part 6: 高频问题视频分析（多次出现的问题视频）")
print("=" * 80)

vid_problem_count = problem.groupby('vid').size().sort_values(ascending=False)
print(f"\n问题视频出现次数分布:")
print(f"  出现1次: {(vid_problem_count == 1).sum()} 个视频")
print(f"  出现2-3次: {((vid_problem_count >= 2) & (vid_problem_count <= 3)).sum()} 个视频")
print(f"  出现4次以上: {(vid_problem_count >= 4).sum()} 个视频")

if len(vid_problem_count) > 0:
    print(f"\n出现次数最多的问题视频 Top5:")
    for vid, cnt in vid_problem_count.head(5).items():
        vid_data = problem[problem['vid'] == vid]
        print(f"  vid={vid}: 出现{cnt}次")
        print(f"    预估ROS={vid_data['pred_ros'].mean():.2f}, 真实ROS={vid_data['real_ros'].mean():.2f}, 偏差={vid_data['ros_bias_pct'].mean():+.1f}%")

# =============================================================================
# Part 7: 结论
# =============================================================================
print("\n" + "=" * 80)
print("Part 7: 综合结论")
print("=" * 80)

print(f"""
【数据基础】
  - 分析时间: {df['dt'].min()} ~ {df['dt'].max()} ({df['dt'].nunique()}天)
  - 分析样本: {len(df)} 条记录，{df['vid'].nunique()} 个视频

【COPC分析】
  - ROS 整体 COPC = {ros_copc:.3f} ({'整体预估偏低' if ros_copc > 1 else '整体预估偏高'})
  - 但存在明显的回归均值问题:
    · 真实ROS < 2: 预估偏高 {df[df['real_ros'] < 2]['ros_bias_pct'].mean():+.0f}%
    · 真实ROS > 6: 预估偏低 {df[df['real_ros'] > 6]['ros_bias_pct'].mean():+.0f}%

【排序效果】
  - 预估排名与真实排名相关系数: {corr_df['corr'].mean():.2f} (±{corr_df['corr'].std():.2f})
  - 排序失准比例: {problem_df['problem_pct'].mean():.1f}% (最高{problem_df['problem_pct'].max():.0f}%)

【问题视频特征】
  - 问题视频占比: {len(problem)/len(df)*100:.1f}%
  - 问题视频ROS偏差: {problem['ros_bias_pct'].mean():+.1f}% (vs 正常视频 {normal['ros_bias_pct'].mean():+.1f}%)
  - 问题视频中真实ROS<2占比: {(problem['real_ros'] < 2).mean()*100:.0f}% (vs 正常视频 {(normal['real_ros'] < 2).mean()*100:.0f}%)

【根本原因】
  模型存在「回归均值」问题，对极端值预估不准确：
  - 低ROS视频被高估 → 排名靠前 → 获得过多曝光
  - 高ROS视频被低估 → 排名靠后 → 曝光不足
""")

# 保存
df.to_csv(output_dir / "v2_深入分析_全量.csv", index=False)
corr_df.to_csv(output_dir / "v2_每日相关系数.csv", index=False)
problem_df.to_csv(output_dir / "v2_每日问题比例.csv", index=False)

print(f"\n数据已保存到 output/ 目录")
