Merge branch 'feature/20260807-xym-update' of algorithm/recommend-emr-dataprocess into feature/20250104-zt-update

ad/02_ad_model_dnn_v11_update.sh

@@ -220,21 +220,27 @@ bucket_feature_from_origin_to_hive() {
 run_pai_flow() {
   local step_start_time=$(date +%s)
 
-  # 并行启动两个 Python 脚本
+  # 并行启动三个 Python 脚本
   python ad/pai_flow_operator.py &
   pid1=$!
 
   python ad/pai_flow_operator_v3.py &
   pid2=$!
 
-  # 等待两个进程完成并获取返回码
+  python ad/pai_flow_operator_v4.py &
+  pid3=$!
+
+  # 等待三个进程完成并获取返回码
   wait $pid1
   return_code1=$?
 
   wait $pid2
   return_code2=$?
 
-  # 检查两个脚本的执行状态
+  wait $pid3
+  return_code3=$?
+
+  # 检查三个脚本的执行状态
   if [ $return_code1 -ne 0 ]; then
     check_run_status $return_code1 $step_start_time "PAI工作流任务1" "PAI工作流任务1执行失败"
     return $return_code1
@@ -245,9 +251,14 @@ run_pai_flow() {
     return $return_code2
   fi
 
-  # 两个任务都成功执行
+  if [ $return_code3 -ne 0 ]; then
+    check_run_status $return_code3 $step_start_time "PAI工作流任务3" "PAI工作流任务3执行失败"
+    return $return_code3
+  fi
+
+  # 三个任务都成功执行
   local duration=$(( $(date +%s) - step_start_time ))
-  log_info "PAI工作流任务1和任务2 执行成功，耗时 ${duration}秒"
+  log_info "PAI工作流任务1、任务2和任务3 执行成功，耗时 ${duration}秒"
   return 0
 }
 

ad/25_xgb_make_data_origin_bucket.sh

@@ -141,7 +141,7 @@ make_bucket_feature_from_origin_to_hive() {
    outputTable:ad_easyrec_train_realtime_data_v3_sampled_v2 \
    inputTable:alg_recsys_ad_sample_all \
    negSampleRate:${neg_sample_rate} \
-   maskFeatureRate:${mask_feature_rate}
+   maskFeatureRate:${mask_feature_rate} &
    local task1=$!
 
   /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
@@ -160,7 +160,7 @@ make_bucket_feature_from_origin_to_hive() {
   negSampleRate:${neg_sample_rate} \
   maskFeatureRate:${mask_feature_rate} \
   bucketFile:20260410_ad_bucket_920.txt \
-  flag:1
+  flag:1 &
   local task2=$!
 
   /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
@@ -179,9 +179,29 @@ make_bucket_feature_from_origin_to_hive() {
     negSampleRate:${neg_sample_rate} \
     maskFeatureRate:${mask_feature_rate} \
     bucketFile:20260410_ad_bucket_920.txt \
-    flag:1
+    flag:1 &
     local task3=$!
 
+  /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+    --class com.aliyun.odps.spark.examples.makedata_ad.v20240718.makedata_ad_33_bucketDataFromOriginToHive_20260808 \
+    --master yarn --driver-memory 2G --executor-memory 3G --executor-cores 1 --num-executors 30 \
+    --conf spark.dynamicAllocation.enabled=true \
+    --conf spark.shuffle.service.enabled=true \
+    --conf spark.dynamicAllocation.maxExecutors=100 \
+    ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+    beginStr:${today_early_1} endStr:${today_early_1} \
+    filterHours:${FILTER_HOURS:-00,01,02,03,04,05} \
+    filterAdverIds:${FILTER_ADVER_IDS} \
+    filterNames:_4h_,_5h_,adid_,targeting_conversion_ \
+    outputTable:ad_easyrec_train_realtime_data_v7_sampled \
+    inputTable:alg_recsys_ad_sample_all_v3 \
+    negSampleRate:${neg_sample_rate} \
+    maskFeatureRate:${mask_feature_rate} \
+    bucketFile:20260807_ad_bucket_1112.txt \
+    tablePart:128 \
+    flag:1 &
+    local task4=$!
+
   wait ${task1}
   local task1_return_code=$?
 
@@ -191,8 +211,12 @@ make_bucket_feature_from_origin_to_hive() {
   wait ${task3}
   local task3_return_code=$?
 
+  wait ${task4}
+  local task4_return_code=$?
+
   check_run_status ${task1_return_code} ${step_start_time} "离线数据spark特征分桶任务"
   check_run_status ${task2_return_code} ${step_start_time} "在线日志spark特征分桶任务"
   check_run_status ${task3_return_code} ${step_start_time} "新特征表spark特征分桶任务"
+  check_run_status ${task4_return_code} ${step_start_time} "v7特征表spark特征分桶任务"
 
 }

ad/pai_flow_operator_v4.py

@@ -0,0 +1,795 @@
+# -*- coding: utf-8 -*-
+import functools
+import os
+import re
+import sys
+import time
+import json
+import pandas as pd
+from alibabacloud_paistudio20210202.client import Client as PaiStudio20210202Client
+from alibabacloud_tea_openapi import models as open_api_models
+from alibabacloud_paistudio20210202 import models as pai_studio_20210202_models
+from alibabacloud_tea_util import models as util_models
+from alibabacloud_tea_util.client import Client as UtilClient
+from alibabacloud_eas20210701.client import Client as eas20210701Client
+from alibabacloud_paiflow20210202 import models as paiflow_20210202_models
+from alibabacloud_paiflow20210202.client import Client as PAIFlow20210202Client
+from datetime import datetime, timedelta
+from odps import ODPS
+from ad_monitor_util import _monitor
+import alibabacloud_oss_v2 as oss
+
+target_names = {
+    '样本shuffle',
+    '模型训练-样本shufle',
+    '模型导出-2',
+    '更新EAS服务(Beta)-1',
+    '虚拟起始节点',
+    '二分类评估-1',
+    '二分类评估-2',
+    '预测结果对比'
+}
+
+EXPERIMENT_ID = "draft-2xa5ge0dt9uzv5ge2y"
+ACCESS_KEY_ID = "LTAI5tFGqgC8f3mh1fRCrAEy"
+ACCESS_KEY_SECRET = "XhOjK9XmTYRhVAtf6yii4s4kZwWzvV"
+
+MAX_RETRIES = 3
+
+
+def retry(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        retries = 0
+        while retries < MAX_RETRIES:
+            try:
+                result = func(*args, **kwargs)
+                if result is not False:
+                    return result
+            except Exception as e:
+                print(f"函数 {func.__name__} 执行时发生异常: {e}，重试第 {retries + 1} 次")
+            retries += 1
+        print(f"函数 {func.__name__} 重试 {MAX_RETRIES} 次后仍失败。")
+        return False
+
+    return wrapper
+
+
+def get_odps_instance(project):
+    odps = ODPS(
+        access_id=ACCESS_KEY_ID,
+        secret_access_key=ACCESS_KEY_SECRET,
+        project=project,
+        endpoint='http://service.cn.maxcompute.aliyun.com/api',
+    )
+    return odps
+
+
+def get_data_from_odps(project, table, num):
+    odps = get_odps_instance(project)
+    try:
+        # 要查询的 SQL 语句
+        sql = f'select * from {table} limit {num}'
+        # 执行 SQL 查询
+        with odps.execute_sql(sql).open_reader() as reader:
+            df = reader.to_pandas()
+            # 查询数量小于目标数量时 返回空
+            if len(df) < num:
+                return None
+            return df
+    except Exception as e:
+        print(f"发生错误: {e}")
+
+
+def get_dict_from_odps(project, table):
+    odps = get_odps_instance(project)
+    try:
+        # 要查询的 SQL 语句
+        sql = f'select * from {table}'
+        # 执行 SQL 查询
+        with odps.execute_sql(sql).open_reader() as reader:
+            data = {}
+            for record in reader:
+                record_list = list(record)
+                key = record_list[0][1]
+                value = record_list[1][1]
+                data[key] = value
+            return data
+    except Exception as e:
+        print(f"发生错误: {e}")
+
+
+def get_dates_between(start_date_str, end_date_str):
+    start_date = datetime.strptime(start_date_str, '%Y%m%d')
+    end_date = datetime.strptime(end_date_str, '%Y%m%d')
+    dates = []
+    current_date = start_date
+    while current_date <= end_date:
+        dates.append(current_date.strftime('%Y%m%d'))
+        current_date += timedelta(days=1)
+    return dates
+
+
+def read_file_to_list():
+    try:
+        current_dir = os.getcwd()
+        file_path = os.path.join(current_dir, 'ad', 'holidays.txt')
+        with open(file_path, 'r', encoding='utf-8') as file:
+            content = file.read()
+            return content.split('\n')
+    except FileNotFoundError:
+        raise Exception(f"错误：未找到 {file_path} 文件。")
+    except Exception as e:
+        raise Exception(f"错误：发生了一个未知错误: {e}")
+    return []
+
+
+def get_previous_days_date(days):
+    current_date = datetime.now()
+    previous_date = current_date - timedelta(days=days)
+    return previous_date.strftime('%Y%m%d')
+
+
+def remove_elements(lst1, lst2):
+    return [element for element in lst1 if element not in lst2]
+
+
+def process_list(lst, append_str):
+    # 给列表中每个元素拼接相同的字符串
+    appended_list = [append_str + element for element in lst]
+    # 将拼接后的列表元素用逗号拼接成一个字符串
+    result_str = ','.join(appended_list)
+    return result_str
+
+
+def get_train_data_list(date_begin):
+    end_date = get_previous_days_date(1)
+    date_list = get_dates_between(date_begin, end_date)
+    filter_date_list = read_file_to_list()
+    date_list = remove_elements(date_list, filter_date_list)
+    return date_list
+
+# 只替换第一次匹配的'where dt in ()'中的日期
+def update_data_date_range(old_str, date_begin='20250605'):
+    date_list = get_train_data_list(date_begin)
+    train_list = ["'" + item + "'" for item in date_list]
+    result = ','.join(train_list)
+    start_index = old_str.find('where dt in (')
+    if start_index != -1:
+        equal_sign_index = start_index + len('where dt in (')
+        # 找到下一个双引号的位置
+        next_quote_index = old_str.find(')', equal_sign_index)
+        if next_quote_index != -1:
+            # 进行替换
+            new_value = old_str[:equal_sign_index] + result + old_str[next_quote_index:]
+            return new_value
+    return None
+
+
+def compare_timestamp_with_today_start(time_str):
+    # 解析时间字符串为 datetime 对象
+    time_obj = datetime.fromisoformat(time_str)
+    # 将其转换为时间戳
+    target_timestamp = time_obj.timestamp()
+    # 获取今天开始的时间
+    today_start = datetime.combine(datetime.now().date(), datetime.min.time())
+    # 将今天开始时间转换为时间戳
+    today_start_timestamp = today_start.timestamp()
+    return target_timestamp > today_start_timestamp
+
+
+def update_train_table(old_str, table):
+    address = 'odps://pai_algo/tables/'
+    train_table = address + table
+    start_index = old_str.find('-Dtrain_tables="')
+    if start_index != -1:
+        # 确定等号的位置
+        equal_sign_index = start_index + len('-Dtrain_tables="')
+        # 找到下一个双引号的位置
+        next_quote_index = old_str.find('"', equal_sign_index)
+        if next_quote_index != -1:
+            # 进行替换
+            new_value = old_str[:equal_sign_index] + train_table + old_str[next_quote_index:]
+            return new_value
+    return None
+
+
+class PAIClient:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def create_client() -> PaiStudio20210202Client:
+        """
+        使用AK&SK初始化账号Client
+        @return: Client
+        @throws Exception
+        """
+        # 工程代码泄露可能会导致 AccessKey 泄露，并威胁账号下所有资源的安全性。以下代码示例仅供参考。
+        # 建议使用更安全的 STS 方式，更多鉴权访问方式请参见：https://help.aliyun.com/document_detail/378659.html。
+        config = open_api_models.Config(
+            access_key_id=ACCESS_KEY_ID,
+            access_key_secret=ACCESS_KEY_SECRET
+        )
+        # Endpoint 请参考 https://api.aliyun.com/product/PaiStudio
+        config.endpoint = f'pai.cn-hangzhou.aliyuncs.com'
+        return PaiStudio20210202Client(config)
+
+    @staticmethod
+    def create_eas_client() -> eas20210701Client:
+        """
+        使用AK&SK初始化账号Client
+        @return: Client
+        @throws Exception
+        """
+        # 工程代码泄露可能会导致 AccessKey 泄露，并威胁账号下所有资源的安全性。以下代码示例仅供参考。
+        # 建议使用更安全的 STS 方式，更多鉴权访问方式请参见：https://help.aliyun.com/document_detail/378659.html。
+        config = open_api_models.Config(
+            access_key_id=ACCESS_KEY_ID,
+            access_key_secret=ACCESS_KEY_SECRET
+        )
+        # Endpoint 请参考 https://api.aliyun.com/product/PaiStudio
+        config.endpoint = f'pai-eas.cn-hangzhou.aliyuncs.com'
+        return eas20210701Client(config)
+
+    @staticmethod
+    def create_flow_client() -> PAIFlow20210202Client:
+        """
+        使用AK&SK初始化账号Client
+        @return: Client
+        @throws Exception
+        """
+        # 工程代码泄露可能会导致 AccessKey 泄露，并威胁账号下所有资源的安全性。以下代码示例仅供参考。
+        # 建议使用更安全的 STS 方式，更多鉴权访问方式请参见：https://help.aliyun.com/document_detail/378659.html。
+        config = open_api_models.Config(
+            # 必填，请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_ID。,
+            access_key_id=ACCESS_KEY_ID,
+            # 必填，请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_SECRET。,
+            access_key_secret=ACCESS_KEY_SECRET
+        )
+        # Endpoint 请参考 https://api.aliyun.com/product/PAIFlow
+        config.endpoint = f'paiflow.cn-hangzhou.aliyuncs.com'
+        return PAIFlow20210202Client(config)
+
+    @staticmethod
+    def get_work_flow_draft_list(workspace_id: str):
+        client = PAIClient.create_client()
+        list_experiments_request = pai_studio_20210202_models.ListExperimentsRequest(
+            workspace_id=workspace_id
+        )
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            resp = client.list_experiments_with_options(list_experiments_request, headers, runtime)
+            return resp.body.to_map()
+        except Exception as error:
+            raise Exception(f"get_work_flow_draft_list error {error}")
+
+    @staticmethod
+    def get_work_flow_draft(experiment_id: str):
+        client = PAIClient.create_client()
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            # 复制代码运行请自行打印 API 的返回值
+            resp = client.get_experiment_with_options(experiment_id, headers, runtime)
+            return resp.body.to_map()
+        except Exception as error:
+            raise Exception(f"get_work_flow_draft error {error}")
+
+    @staticmethod
+    def get_describe_service(service_name: str):
+        client = PAIClient.create_eas_client()
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            # 复制代码运行请自行打印 API 的返回值
+            resp = client.describe_service_with_options('cn-hangzhou', service_name, headers, runtime)
+            return resp.body.to_map()
+        except Exception as error:
+            raise Exception(f"get_describe_service error {error}")
+
+    @staticmethod
+    def update_experiment_content(experiment_id: str, content: str, version: int):
+        client = PAIClient.create_client()
+        update_experiment_content_request = pai_studio_20210202_models.UpdateExperimentContentRequest(content=content,
+                                                                                                      version=version)
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            # 复制代码运行请自行打印 API 的返回值
+            resp = client.update_experiment_content_with_options(experiment_id, update_experiment_content_request,
+                                                                 headers, runtime)
+            print(resp.body.to_map())
+        except Exception as error:
+            raise Exception(f"update_experiment_content error {error}")
+
+    @staticmethod
+    def create_job(experiment_id: str, node_id: str, execute_type: str):
+        client = PAIClient.create_client()
+        create_job_request = pai_studio_20210202_models.CreateJobRequest()
+        create_job_request.experiment_id = experiment_id
+        create_job_request.node_id = node_id
+        create_job_request.execute_type = execute_type
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            # 复制代码运行请自行打印 API 的返回值
+            resp = client.create_job_with_options(create_job_request, headers, runtime)
+            return resp.body.to_map()
+        except Exception as error:
+            raise Exception(f"create_job error {error}")
+
+    @staticmethod
+    def get_jobs_list(experiment_id: str, order='DESC'):
+        client = PAIClient.create_client()
+        list_jobs_request = pai_studio_20210202_models.ListJobsRequest(
+            experiment_id=experiment_id,
+            order=order
+        )
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            # 复制代码运行请自行打印 API 的返回值
+            resp = client.list_jobs_with_options(list_jobs_request, headers, runtime)
+            return resp.body.to_map()
+        except Exception as error:
+            raise Exception(f"get_jobs_list error {error}")
+
+    @staticmethod
+    def get_job_detail(job_id: str, verbose=False):
+        client = PAIClient.create_client()
+        get_job_request = pai_studio_20210202_models.GetJobRequest(
+            verbose=verbose
+        )
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            # 复制代码运行请自行打印 API 的返回值
+            resp = client.get_job_with_options(job_id, get_job_request, headers, runtime)
+            return resp.body.to_map()
+        except Exception as error:
+            # 此处仅做打印展示，请谨慎对待异常处理，在工程项目中切勿直接忽略异常。
+            # 错误 message
+            print(error.message)
+            # 诊断地址
+            print(error.data.get("Recommend"))
+            UtilClient.assert_as_string(error.message)
+
+    @staticmethod
+    def get_flow_out_put(pipeline_run_id: str, node_id: str, depth: int):
+        client = PAIClient.create_flow_client()
+        list_pipeline_run_node_outputs_request = paiflow_20210202_models.ListPipelineRunNodeOutputsRequest(
+            depth=depth
+        )
+        runtime = util_models.RuntimeOptions()
+        headers = {}
+        try:
+            # 复制代码运行请自行打印 API 的返回值
+            resp = client.list_pipeline_run_node_outputs_with_options(pipeline_run_id, node_id,
+                                                                      list_pipeline_run_node_outputs_request, headers,
+                                                                      runtime)
+            return resp.body.to_map()
+        except Exception as error:
+            # 此处仅做打印展示，请谨慎对待异常处理，在工程项目中切勿直接忽略异常。
+            # 错误 message
+            print(error.message)
+            # 诊断地址
+            print(error.data.get("Recommend"))
+            UtilClient.assert_as_string(error.message)
+
+
+def extract_date_yyyymmdd(input_string):
+    pattern = r'\d{8}'
+    matches = re.findall(pattern, input_string)
+    if matches:
+        return matches[0]
+    return None
+
+def get_online_model_config(service_name: str):
+    model_config = {}
+    model_detail = PAIClient.get_describe_service(service_name)
+    service_config_str = model_detail['ServiceConfig']
+    service_config = json.loads(service_config_str)
+    model_path = service_config['model_path']
+    model_config['model_path'] = model_path
+    online_date = extract_date_yyyymmdd(model_path)
+    model_config['online_date'] = online_date
+    return model_config
+
+
+def update_shuffle_flow(table):
+    draft = PAIClient.get_work_flow_draft(EXPERIMENT_ID)
+    print(json.dumps(draft, ensure_ascii=False))
+    content = draft['Content']
+    version = draft['Version']
+    content_json = json.loads(content)
+    nodes = content_json.get('nodes')
+    for node in nodes:
+        name = node['name']
+        if name == '模型训练-样本shufle':
+            properties = node['properties']
+            for property in properties:
+                if property['name'] == 'sql':
+                    value = property['value']
+                    new_value = update_train_table(value, table)
+                    if new_value is None:
+                        print("error")
+                    property['value'] = new_value
+    new_content = json.dumps(content_json, ensure_ascii=False)
+    PAIClient.update_experiment_content(EXPERIMENT_ID, new_content, version)
+
+
+def update_shuffle_flow_1():
+    draft = PAIClient.get_work_flow_draft(EXPERIMENT_ID)
+    print(json.dumps(draft, ensure_ascii=False))
+    content = draft['Content']
+    version = draft['Version']
+    print(content)
+    content_json = json.loads(content)
+    nodes = content_json.get('nodes')
+    for node in nodes:
+        name = node['name']
+        if name == '模型训练-样本shufle':
+            properties = node['properties']
+            for property in properties:
+                if property['name'] == 'sql':
+                    value = property['value']
+                    new_value = update_data_date_range(value)
+                    if new_value is None:
+                        print("error")
+                    property['value'] = new_value
+    new_content = json.dumps(content_json, ensure_ascii=False)
+    PAIClient.update_experiment_content(EXPERIMENT_ID, new_content, version)
+
+
+def wait_job_end(job_id: str, check_interval=300):
+    while True:
+        job_detail = PAIClient.get_job_detail(job_id)
+        print(job_detail)
+        statue = job_detail['Status']
+        # Initialized: 初始化完成 Starting：开始 WorkflowServiceStarting：准备提交 Running：运行中 ReadyToSchedule：准备运行（前序节点未完成导致）
+        if (statue == 'Initialized' or statue == 'Starting' or statue == 'WorkflowServiceStarting'
+                or statue == 'Running' or statue == 'ReadyToSchedule'):
+            time.sleep(check_interval)
+            continue
+        # Failed：运行失败 Terminating：终止中 Terminated：已终止 Unknown：未知 Skipped：跳过（前序节点失败导致） Succeeded：运行成功
+        if statue == 'Failed' or statue == 'Terminating' or statue == 'Unknown' or statue == 'Skipped' or statue == 'Succeeded':
+            return job_detail
+
+
+def get_node_dict():
+    draft = PAIClient.get_work_flow_draft(EXPERIMENT_ID)
+    content = draft['Content']
+    content_json = json.loads(content)
+    nodes = content_json.get('nodes')
+    node_dict = {}
+    for node in nodes:
+        name = node['name']
+        # 检查名称是否在目标名称集合中
+        if name in target_names:
+            node_dict[name] = node['id']
+    return node_dict
+
+
+def get_job_dict():
+    job_dict = {}
+    jobs_list = PAIClient.get_jobs_list(EXPERIMENT_ID)
+    for job in jobs_list['Jobs']:
+        # 解析时间字符串为 datetime 对象
+        if not compare_timestamp_with_today_start(job['GmtCreateTime']):
+            break
+        job_id = job['JobId']
+        job_detail = PAIClient.get_job_detail(job_id, verbose=True)
+        for name in target_names:
+            if job_detail['Status'] != 'Succeeded':
+                continue
+            if name in job_dict:
+                continue
+            if name in job_detail['RunInfo']:
+                job_dict[name] = job_detail['JobId']
+    return job_dict
+
+@retry
+def update_online_flow():
+    try:
+        online_model_config = get_online_model_config('ad_rank_dnn_v11_easyrec_v3')
+        draft = PAIClient.get_work_flow_draft(EXPERIMENT_ID)
+        print(json.dumps(draft, ensure_ascii=False))
+        content = draft['Content']
+        version = draft['Version']
+        print(content)
+        content_json = json.loads(content)
+        nodes = content_json.get('nodes')
+        global_params = content_json.get('globalParams')
+        bizdate = get_previous_days_date(1)
+        for global_param in global_params:
+            try:
+                if global_param['name'] == 'bizdate':
+                    global_param['value'] = bizdate
+                if global_param['name'] == 'online_version_dt':
+                    global_param['value'] = online_model_config['online_date']
+                if global_param['name'] == 'eval_date':
+                    global_param['value'] = bizdate
+                if global_param['name'] == 'online_model_path':
+                    global_param['value'] = online_model_config['model_path']
+            except KeyError:
+                raise Exception("在处理全局参数时，字典中缺少必要的键")
+        for node in nodes:
+            try:
+                name = node['name']
+                if name in ('样本shuffle',):
+                    date_begin = get_previous_days_date(90) if name == '样本shuffle' else get_previous_days_date(10)
+                    properties = node['properties']
+                    for property in properties:
+                        if property['name'] == 'sql':
+                            value = property['value']
+                            new_value = update_data_date_range(value, date_begin)
+                            if new_value is None:
+                                print("error")
+                            property['value'] = new_value
+            except KeyError:
+                raise Exception("在处理节点属性时，字典中缺少必要的键")
+        new_content = json.dumps(content_json, ensure_ascii=False)
+        PAIClient.update_experiment_content(EXPERIMENT_ID, new_content, version)
+        return True
+    except json.JSONDecodeError:
+        raise Exception("JSON 解析错误，可能是草稿内容格式不正确")
+    except Exception as e:
+        raise Exception(f"发生未知错误: {e}")
+
+@retry
+def shuffle_table():
+    try:
+        node_dict = get_node_dict()
+        train_node_id = node_dict['样本shuffle']
+        execute_type = 'EXECUTE_FROM_HERE'
+        validate_res = PAIClient.create_job(EXPERIMENT_ID, train_node_id, execute_type)
+        validate_job_id = validate_res['JobId']
+        validate_job_detail = wait_job_end(validate_job_id, 10)
+        if validate_job_detail['Status'] == 'Succeeded':
+            return True
+        return False
+    except Exception as e:
+        error_message = f"在执行 shuffle_table 函数时发生异常: {str(e)}"
+        print(error_message)
+        raise Exception(error_message)
+
+
+@retry
+def shuffle_train_model():
+    try:
+        node_dict = get_node_dict()
+        job_dict = get_job_dict()
+        job_id = job_dict['样本shuffle']
+        validate_job_detail = wait_job_end(job_id)
+        if validate_job_detail['Status'] == 'Succeeded':
+            pipeline_run_id = validate_job_detail['RunId']
+            node_id = validate_job_detail['PaiflowNodeId']
+            flow_out_put_detail = PAIClient.get_flow_out_put(pipeline_run_id, node_id, 2)
+            outputs = flow_out_put_detail['Outputs']
+            table = None
+            for output in outputs:
+                if output["Producer"] == node_dict['样本shuffle'] and output["Name"] == "outputTable":
+                    value1 = json.loads(output["Info"]['value'])
+                    table = value1['location']['table']
+            if table is not None:
+                update_shuffle_flow(table)
+                node_dict = get_node_dict()
+                train_node_id = node_dict['模型训练-样本shufle']
+                execute_type = 'EXECUTE_ONE'
+                train_res = PAIClient.create_job(EXPERIMENT_ID, train_node_id, execute_type)
+                train_job_id = train_res['JobId']
+                train_job_detail = wait_job_end(train_job_id)
+                if train_job_detail['Status'] == 'Succeeded':
+                    return True
+        return False
+    except Exception as e:
+        error_message = f"在执行 shuffle_train_model 函数时发生异常: {str(e)}"
+        print(error_message)
+        raise Exception(error_message)
+
+
+@retry
+def export_model():
+    try:
+        node_dict = get_node_dict()
+        export_node_id = node_dict['模型导出-2']
+        execute_type = 'EXECUTE_ONE'
+        export_res = PAIClient.create_job(EXPERIMENT_ID, export_node_id, execute_type)
+        export_job_id = export_res['JobId']
+        export_job_detail = wait_job_end(export_job_id)
+        if export_job_detail['Status'] == 'Succeeded':
+            return True
+        return False
+    except Exception as e:
+        error_message = f"在执行 export_model 函数时发生异常: {str(e)}"
+        print(error_message)
+        raise Exception(error_message)
+
+
+def update_online_model():
+    try:
+        node_dict = get_node_dict()
+        train_node_id = node_dict['更新EAS服务(Beta)-1']
+        execute_type = 'EXECUTE_ONE'
+        train_res = PAIClient.create_job(EXPERIMENT_ID, train_node_id, execute_type)
+        train_job_id = train_res['JobId']
+        train_job_detail = wait_job_end(train_job_id)
+        if train_job_detail['Status'] == 'Succeeded':
+            return True
+        return False
+    except Exception as e:
+        error_message = f"在执行 update_online_model 函数时发生异常: {str(e)}"
+        print(error_message)
+        raise Exception(error_message)
+
+@retry
+def get_validate_model_data():
+    try:
+        node_dict = get_node_dict()
+        train_node_id = node_dict['虚拟起始节点']
+        execute_type = 'EXECUTE_FROM_HERE'
+        validate_res = PAIClient.create_job(EXPERIMENT_ID, train_node_id, execute_type)
+        validate_job_id = validate_res['JobId']
+        validate_job_detail = wait_job_end(validate_job_id)
+        if validate_job_detail['Status'] == 'Succeeded':
+            return True
+        return False
+    except Exception as e:
+        error_message = f"在执行 get_validate_model_data 函数时出现异常: {e}"
+        print(error_message)
+        raise Exception(error_message)
+
+
+def validate_model_data_accuracy():
+    try:
+        table_dict = {}
+        node_dict = get_node_dict()
+        job_dict = get_job_dict()
+        job_id = job_dict['虚拟起始节点']
+        validate_job_detail = wait_job_end(job_id)
+        if validate_job_detail['Status'] == 'Succeeded':
+            pipeline_run_id = validate_job_detail['RunId']
+            node_id = validate_job_detail['PaiflowNodeId']
+            flow_out_put_detail = PAIClient.get_flow_out_put(pipeline_run_id, node_id, 3)
+            print(flow_out_put_detail)
+            outputs = flow_out_put_detail['Outputs']
+            for output in outputs:
+                if output["Producer"] == node_dict['二分类评估-1'] and output["Name"] == "outputMetricTable":
+                    value1 = json.loads(output["Info"]['value'])
+                    table_dict['二分类评估-1'] = value1['location']['table']
+                if output["Producer"] == node_dict['二分类评估-2'] and output["Name"] == "outputMetricTable":
+                    value2 = json.loads(output["Info"]['value'])
+                    table_dict['二分类评估-2'] = value2['location']['table']
+                if output["Producer"] == node_dict['预测结果对比'] and output["Name"] == "outputTable":
+                    value3 = json.loads(output["Info"]['value'])
+                    table_dict['预测结果对比'] = value3['location']['table']
+        num = 10
+        df = get_data_from_odps('pai_algo', table_dict['预测结果对比'], 10)
+        # 对指定列取绝对值再求和
+        old_abs_avg = df['old_error'].abs().sum() / num
+        new_abs_avg = df['new_error'].abs().sum() / num
+        new_auc = get_dict_from_odps('pai_algo', table_dict['二分类评估-1'])['AUC']
+        old_auc = get_dict_from_odps('pai_algo', table_dict['二分类评估-2'])['AUC']
+        bizdate = get_previous_days_date(1)
+        score_diff = abs(old_abs_avg - new_abs_avg)
+        msg = ""
+        result = False
+        if new_abs_avg > 0.1:
+            msg += f'DNN线上模型评估{bizdate}的数据，绝对误差大于0.1，请检查'
+            level = 'error'
+        elif score_diff > 0.02 and new_abs_avg - old_abs_avg > 0.02:
+            msg += f'DNN两个模型评估${bizdate}的数据，两个模型分数差异为: ${score_diff}, 大于0.02, 请检查'
+            level = 'error'
+        else:
+            msg += 'DNN广告模型更新完成'
+            level = 'info'
+            result = True
+
+        # 初始化表格头部
+        top10_msg = "| CID  | 老模型相对真实CTCVR的变化 | 新模型相对真实CTCVR的变化 |"
+        top10_msg += "\n| ---- | --------- | -------- |"
+
+        for index, row in df.iterrows():
+            # 获取指定列的元素
+            cid = row['cid']
+            old_error = row['old_error']
+            new_error = row['new_error']
+            top10_msg += f"\n| {int(cid)} | {old_error} | {new_error} | "
+        print(top10_msg)
+        msg += f"\n\t - 老模型AUC: {old_auc}"
+        msg += f"\n\t - 新模型AUC: {new_auc}"
+        msg += f"\n\t - 老模型Top10差异平均值: {old_abs_avg}"
+        msg += f"\n\t - 新模型Top10差异平均值: {new_abs_avg}"
+        return result, msg, level, top10_msg
+
+    except Exception as e:
+        error_message = f"在执行 validate_model_data_accuracy 函数时出现异常: {str(e)}"
+        print(error_message)
+        raise Exception(error_message)
+
+
+def update_trained_cids_pointer(model_name=None, dt_version=None):
+    # 如均为空，则从工作流中获取
+    if not model_name and not dt_version:
+        draft = PAIClient.get_work_flow_draft(EXPERIMENT_ID)
+        content = draft['Content']
+        content_json = json.loads(content)
+        global_params = content_json.get('globalParams', [])
+        model_name = None
+        dt_version = None
+        for param in global_params:
+            if param.get('name') == 'model_name':
+                model_name = param.get('value')
+            if param.get('name') == 'bizdate':
+                dt_version = param.get('value')
+        if not model_name or not dt_version:
+            raise Exception("globalParams 中未找到 model_name 或 bizdate")
+    elif not (model_name and dt_version):
+        # 不允许其中一个为空
+        raise Exception("model_name 和 dt_version 必须同时提供")
+    model_version = {}
+    model_version['modelName'] = f"model_name={model_name}"
+    model_version['dtVersion'] = f"dt_version={dt_version}"
+    model_version['timestamp'] = int(time.time())
+    print(json.dumps(model_version, ensure_ascii=False, indent=4).encode('utf-8'))
+    bucket_name = "art-recommend"
+    object_key = "fengzhoutian/pai_model_trained_cids/model_version_v3.json"
+
+    oss_config = oss.config.load_default()
+    oss_config.credentials_provider = oss.credentials.StaticCredentialsProvider(
+        access_key_id=ACCESS_KEY_ID, access_key_secret=ACCESS_KEY_SECRET
+    )
+    oss_config.region = "cn-hangzhou"
+    client = oss.Client(oss_config)
+    ret = client.put_object(oss.PutObjectRequest(
+        bucket=bucket_name,
+        key=object_key,
+        body=json.dumps(model_version, ensure_ascii=False, indent=4).encode('utf-8')
+    ))
+    print(f'oss put status code: {ret.status_code},'
+          f' request id: {ret.request_id},'
+          f' content md5: {ret.content_md5},'
+          f' etag: {ret.etag},'
+          f' hash crc64: {ret.hash_crc64},'
+          f' version id: {ret.version_id},'
+          f' content: {model_version}'
+          )
+
+
+if __name__ == '__main__':
+    start_time = int(time.time())
+    functions = [update_online_flow, shuffle_table, shuffle_train_model, export_model, get_validate_model_data]
+    function_names = [func.__name__ for func in functions]
+
+    start_function = None
+    if len(sys.argv) > 1:
+        start_function = sys.argv[1]
+        if start_function not in function_names:
+            print(f"指定的起始函数 {start_function} 不存在，请选择以下函数之一：{', '.join(function_names)}")
+            sys.exit(1)
+
+    start_index = 0
+    if start_function:
+        start_index = function_names.index(start_function)
+
+    for func in functions[start_index:]:
+        if not func():
+            print(f"{func.__name__} 执行失败，后续函数不再执行。")
+            step_end_time = int(time.time())
+            elapsed = step_end_time - start_time
+            _monitor('error', f"DNN模型更新，{func.__name__} 执行失败，后续函数不再执行，请检查", start_time, elapsed, None)
+            break
+    else:
+        print("所有函数都成功执行，可以继续下一步操作。")
+        result, msg, level, top10_msg = validate_model_data_accuracy()
+        if result:
+            update_online_res = update_online_model()
+            if update_online_res:
+                update_trained_cids_pointer()
+                print("success")
+        step_end_time = int(time.time())
+        elapsed = step_end_time - start_time
+        print(level, msg, start_time, elapsed, top10_msg)
+        _monitor(level, msg, start_time, elapsed, top10_msg)

src/main/resources/20260807_ad_feature_name.txt

@@ -0,0 +1,192 @@
+k1_2h_view
+k1_2h_click
+k1_2h_conver
+k1_2h_ctr
+k1_2h_cvr
+k1_2h_ctvr
+k1_4h_view
+k1_4h_click
+k1_4h_conver
+k1_4h_ctr
+k1_4h_cvr
+k1_4h_ctvr
+k1_6h_view
+k1_6h_click
+k1_6h_conver
+k1_6h_ctr
+k1_6h_cvr
+k1_6h_ctvr
+k1_12h_view
+k1_12h_click
+k1_12h_conver
+k1_12h_ctr
+k1_12h_cvr
+k1_12h_ctvr
+k1_1d_view
+k1_1d_click
+k1_1d_conver
+k1_1d_ctr
+k1_1d_cvr
+k1_1d_ctvr
+k1_3d_view
+k1_3d_click
+k1_3d_conver
+k1_3d_ctr
+k1_3d_cvr
+k1_3d_ctvr
+k1_today_view
+k1_today_click
+k1_today_conver
+k1_today_ctr
+k1_today_cvr
+k1_today_ctvr
+k1_1w_view
+k1_1w_click
+k1_1w_conver
+k1_1w_ctr
+k1_1w_cvr
+k1_1w_ctvr
+k2_2h_view
+k2_2h_click
+k2_2h_conver
+k2_2h_ctr
+k2_2h_cvr
+k2_2h_ctvr
+k2_4h_view
+k2_4h_click
+k2_4h_conver
+k2_4h_ctr
+k2_4h_cvr
+k2_4h_ctvr
+k2_6h_view
+k2_6h_click
+k2_6h_conver
+k2_6h_ctr
+k2_6h_cvr
+k2_6h_ctvr
+k2_12h_view
+k2_12h_click
+k2_12h_conver
+k2_12h_ctr
+k2_12h_cvr
+k2_12h_ctvr
+k2_1d_view
+k2_1d_click
+k2_1d_conver
+k2_1d_ctr
+k2_1d_cvr
+k2_1d_ctvr
+k2_3d_view
+k2_3d_click
+k2_3d_conver
+k2_3d_ctr
+k2_3d_cvr
+k2_3d_ctvr
+k2_today_view
+k2_today_click
+k2_today_conver
+k2_today_ctr
+k2_today_cvr
+k2_today_ctvr
+k2_1w_view
+k2_1w_click
+k2_1w_conver
+k2_1w_ctr
+k2_1w_cvr
+k2_1w_ctvr
+k3_2h_view
+k3_2h_click
+k3_2h_conver
+k3_2h_ctr
+k3_2h_cvr
+k3_2h_ctvr
+k3_4h_view
+k3_4h_click
+k3_4h_conver
+k3_4h_ctr
+k3_4h_cvr
+k3_4h_ctvr
+k3_6h_view
+k3_6h_click
+k3_6h_conver
+k3_6h_ctr
+k3_6h_cvr
+k3_6h_ctvr
+k3_12h_view
+k3_12h_click
+k3_12h_conver
+k3_12h_ctr
+k3_12h_cvr
+k3_12h_ctvr
+k3_1d_view
+k3_1d_click
+k3_1d_conver
+k3_1d_ctr
+k3_1d_cvr
+k3_1d_ctvr
+k3_3d_view
+k3_3d_click
+k3_3d_conver
+k3_3d_ctr
+k3_3d_cvr
+k3_3d_ctvr
+k3_today_view
+k3_today_click
+k3_today_conver
+k3_today_ctr
+k3_today_cvr
+k3_today_ctvr
+k3_1w_view
+k3_1w_click
+k3_1w_conver
+k3_1w_ctr
+k3_1w_cvr
+k3_1w_ctvr
+k4_2h_view
+k4_2h_click
+k4_2h_conver
+k4_2h_ctr
+k4_2h_cvr
+k4_2h_ctvr
+k4_4h_view
+k4_4h_click
+k4_4h_conver
+k4_4h_ctr
+k4_4h_cvr
+k4_4h_ctvr
+k4_6h_view
+k4_6h_click
+k4_6h_conver
+k4_6h_ctr
+k4_6h_cvr
+k4_6h_ctvr
+k4_12h_view
+k4_12h_click
+k4_12h_conver
+k4_12h_ctr
+k4_12h_cvr
+k4_12h_ctvr
+k4_1d_view
+k4_1d_click
+k4_1d_conver
+k4_1d_ctr
+k4_1d_cvr
+k4_1d_ctvr
+k4_3d_view
+k4_3d_click
+k4_3d_conver
+k4_3d_ctr
+k4_3d_cvr
+k4_3d_ctvr
+k4_today_view
+k4_today_click
+k4_today_conver
+k4_today_ctr
+k4_today_cvr
+k4_today_ctvr
+k4_1w_view
+k4_1w_click
+k4_1w_conver
+k4_1w_ctr
+k4_1w_cvr
+k4_1w_ctvr

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_31_originData_20260807.scala

@@ -0,0 +1,498 @@
+package com.aliyun.odps.spark.examples.makedata_ad.v20240718
+
+import com.alibaba.fastjson.{JSON, JSONObject}
+import com.aliyun.odps.TableSchema
+import com.aliyun.odps.data.Record
+import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, MyHdfsUtils, ParamUtils, env}
+import examples.extractor.RankExtractorFeature_20240530
+import examples.utils.{AdUtil, DateTimeUtil}
+import org.apache.hadoop.io.compress.GzipCodec
+import org.apache.spark.sql.SparkSession
+import org.xm.Similarity
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
+
+/*
+   原始特征处理20260807版本，基于20250110修改
+   * 增加 k1~k4 事件特征，供后续分桶边界计算
+ */
+
+object makedata_ad_31_originData_20260807 {
+  val WILSON_ZSCORE = 1.96
+  val CTR_SMOOTH_BETA_FACTOR = 25
+  val CVR_SMOOTH_BETA_FACTOR = 10
+  val CTCVR_SMOOTH_BETA_FACTOR = 100
+
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession
+      .builder()
+      .appName(this.getClass.getName)
+      .getOrCreate()
+    val sc = spark.sparkContext
+
+    // 1 读取参数
+    val param = ParamUtils.parseArgs(args)
+    val tablePart = param.getOrElse("tablePart", "64").toInt
+    val beginStr = param.getOrElse("beginStr", "2024062008")
+    val endStr = param.getOrElse("endStr", "2024062023")
+    val savePath = param.getOrElse("savePath", "/dw/recommend/model/31_ad_sample_data/")
+    val project = param.getOrElse("project", "loghubods")
+    val table = param.getOrElse("table", "alg_recsys_ad_sample_all_v3")
+    val repartition = param.getOrElse("repartition", "100").toInt
+    val filterHours = param.getOrElse("filterHours", "00,01,02,03,04,05").split(",").toSet
+    val idDefaultValue = param.getOrElse("idDefaultValue", "1.0").toDouble
+    // 2 读取odps+表信息
+    val odpsOps = env.getODPS(sc)
+
+    // 3 循环执行数据生产
+    val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
+    for (dt_hh <- timeRange) {
+      val dt = dt_hh.substring(0, 8)
+      val hh = dt_hh.substring(8, 10)
+      val partition = s"dt=$dt,hh=$hh"
+      if (filterHours.nonEmpty && filterHours.contains(hh)) {
+        println("不执行partiton:" + partition)
+      } else {
+        println("开始执行partiton:" + partition)
+        val odpsData = odpsOps.readTable(project = project,
+            table = table,
+            partition = partition,
+            transfer = func,
+            numPartition = tablePart)
+          .filter(record => {
+            AdUtil.isApi(record)
+          })
+          .map(record => {
+
+            val ts = record.getString("ts").toInt
+            val cid = record.getString("cid")
+            val apptype = record.getString("apptype")
+            val extend: JSONObject = if (record.isNull("extend")) new JSONObject() else
+              JSON.parseObject(record.getString("extend"))
+
+            val featureMap = new JSONObject()
+
+            val b1: JSONObject = if (record.isNull("b1_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b1_feature"))
+            val b2: JSONObject = if (record.isNull("b2_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b2_feature"))
+            val b3: JSONObject = if (record.isNull("b3_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b3_feature"))
+            val b4: JSONObject = if (record.isNull("b4_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b4_feature"))
+            val b5: JSONObject = if (record.isNull("b5_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b5_feature"))
+            val b6: JSONObject = if (record.isNull("b6_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b6_feature"))
+            val b7: JSONObject = if (record.isNull("b7_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b7_feature"))
+            val b8: JSONObject = if (record.isNull("b8_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b8_feature"))
+            val b9: JSONObject = if (record.isNull("b9_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("b9_feature"))
+            val k1: JSONObject = getJsonObject(record, "k1_feature")
+            val k2: JSONObject = getJsonObject(record, "k2_feature")
+            val k3: JSONObject = getJsonObject(record, "k3_feature")
+            val k4: JSONObject = getJsonObject(record, "k4_feature")
+
+
+            featureMap.put("cid_" + cid, idDefaultValue)
+            if (b1.containsKey("adid") && b1.getString("adid").nonEmpty) {
+              featureMap.put("adid_" + b1.getString("adid"), idDefaultValue)
+            }
+            if (b1.containsKey("adverid") && b1.getString("adverid").nonEmpty) {
+              featureMap.put("adverid_" + b1.getString("adverid"), idDefaultValue)
+            }
+            if (b1.containsKey("targeting_conversion") && b1.getString("targeting_conversion").nonEmpty) {
+              featureMap.put("targeting_conversion_" + b1.getString("targeting_conversion"), idDefaultValue)
+            }
+
+            val hour = DateTimeUtil.getHourByTimestamp(ts)
+            featureMap.put("hour_" + hour, idDefaultValue)
+
+            val dayOfWeek = DateTimeUtil.getDayOrWeekByTimestamp(ts)
+            featureMap.put("dayofweek_" + dayOfWeek, idDefaultValue);
+
+            featureMap.put("apptype_" + apptype, idDefaultValue);
+
+            if (extend.containsKey("abcode") && extend.getString("abcode").nonEmpty) {
+              featureMap.put("abcode_" + extend.getString("abcode"), idDefaultValue)
+            }
+
+
+            if (b1.containsKey("cpa")) {
+              featureMap.put("cpa", b1.getString("cpa").toDouble)
+            }
+            if (b1.containsKey("weight") && b1.getString("weight").nonEmpty) {
+              featureMap.put("weight", b1.getString("weight").toDouble)
+            }
+
+            for ((bn, prefix1) <- List(
+              (b2, "b2"), (b3, "b3"), (b4, "b4"), (b5, "b5"), (b8, "b8"), (b9, "b9")
+            )) {
+              for (prefix2 <- List(
+                "1h", "2h", "3h", "4h", "5h", "6h", "12h", "1d", "3d", "7d", "today", "yesterday"
+              )) {
+                val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                // NOTE(zhoutian):
+                // 这里cpc只是为了计算cpm的平滑的工具量，没有实际业务意义，因为cpm并非比率，本身不适合直接计算Wilson平滑
+                // 不使用cpa的原因是未来可能出现广告采用cpc计费的情况或者无法获取转化量的情况，用点击更为稳定
+                // 其它几组特征亦采用相同逻辑
+                // 2025-02-17改为增加固定分母平滑，income实际已经可以直接参与cpm平滑计算
+                val cpc = if (click == 0) 0D else income / click
+                val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                val f4 = conver
+                val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver*log(view)", conver * RankExtractorFeature_20240530.calLog(view))
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver*ctcvr", conver * f2)
+              }
+            }
+
+            for ((bn, prefix1) <- List(
+              (b6, "b6"), (b7, "b7")
+            )) {
+              for (prefix2 <- List(
+                "7d", "14d"
+              )) {
+                val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                val cpc = if (click == 0) 0D else income / click
+                val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                val f4 = conver
+                val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver*log(view)", conver * RankExtractorFeature_20240530.calLog(view))
+                featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver*ctcvr", conver * f2)
+              }
+            }
+
+            val c1: JSONObject = if (record.isNull("c1_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("c1_feature"))
+
+            val midActionList = if (c1.containsKey("action") && c1.getString("action").nonEmpty) {
+              c1.getString("action").split(",").map(r => {
+                val rList = r.split(":")
+                (rList(0), (rList(1).toInt, rList(2).toInt, rList(3).toInt, rList(4).toInt, rList(5)))
+              }).sortBy(-_._2._1).toList
+            } else {
+              new ArrayBuffer[(String, (Int, Int, Int, Int, String))]().toList
+            }
+            // u特征
+            val viewAll = midActionList.size.toDouble
+            val clickAll = midActionList.map(_._2._2).sum.toDouble
+            val converAll = midActionList.map(_._2._3).sum.toDouble
+            val incomeAll = midActionList.map(_._2._4).sum.toDouble
+            featureMap.put("viewAll", viewAll)
+            featureMap.put("clickAll", clickAll)
+            featureMap.put("converAll", converAll)
+            featureMap.put("incomeAll", incomeAll)
+            featureMap.put("ctr_all", RankExtractorFeature_20240530.calDiv(clickAll, viewAll))
+            featureMap.put("ctcvr_all", RankExtractorFeature_20240530.calDiv(converAll, viewAll))
+            featureMap.put("cvr_all", RankExtractorFeature_20240530.calDiv(clickAll, converAll))
+            featureMap.put("ecpm_all", RankExtractorFeature_20240530.calDiv(incomeAll * 1000, viewAll))
+
+            // ui特征
+            val midTimeDiff = scala.collection.mutable.Map[String, Double]()
+            midActionList.foreach {
+              case (cid, (ts_history, click, conver, income, title)) =>
+                if (!midTimeDiff.contains("timediff_view_" + cid)) {
+                  midTimeDiff.put("timediff_view_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                }
+                if (!midTimeDiff.contains("timediff_click_" + cid) && click > 0) {
+                  midTimeDiff.put("timediff_click_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                }
+                if (!midTimeDiff.contains("timediff_conver_" + cid) && conver > 0) {
+                  midTimeDiff.put("timediff_conver_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                }
+            }
+
+            val midActionStatic = scala.collection.mutable.Map[String, Double]()
+            midActionList.foreach {
+              case (cid, (ts_history, click, conver, income, title)) =>
+                midActionStatic.put("actionstatic_view_" + cid, 1.0 + midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0))
+                midActionStatic.put("actionstatic_click_" + cid, click + midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0))
+                midActionStatic.put("actionstatic_conver_" + cid, conver + midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0))
+                midActionStatic.put("actionstatic_income_" + cid, income + midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0))
+            }
+
+            if (midTimeDiff.contains("timediff_view_" + cid)) {
+              featureMap.put("timediff_view", midTimeDiff.getOrDefault("timediff_view_" + cid, 0.0))
+            }
+            if (midTimeDiff.contains("timediff_click_" + cid)) {
+              featureMap.put("timediff_click", midTimeDiff.getOrDefault("timediff_click_" + cid, 0.0))
+            }
+            if (midTimeDiff.contains("timediff_conver_" + cid)) {
+              featureMap.put("timediff_conver", midTimeDiff.getOrDefault("timediff_conver_" + cid, 0.0))
+            }
+            if (midActionStatic.contains("actionstatic_view_" + cid)) {
+              featureMap.put("actionstatic_view", midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0))
+            }
+            if (midActionStatic.contains("actionstatic_click_" + cid)) {
+              featureMap.put("actionstatic_click", midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0))
+            }
+            if (midActionStatic.contains("actionstatic_conver_" + cid)) {
+              featureMap.put("actionstatic_conver", midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0))
+            }
+            if (midActionStatic.contains("actionstatic_income_" + cid)) {
+              featureMap.put("actionstatic_income", midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0))
+            }
+            if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
+              featureMap.put("actionstatic_ctr", RankExtractorFeature_20240530.calDiv(
+                midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+              ))
+            }
+            if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_conver_" + cid)) {
+              featureMap.put("actionstatic_ctcvr", RankExtractorFeature_20240530.calDiv(
+                midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
+                midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+              ))
+            }
+            if (midActionStatic.contains("actionstatic_conver_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
+              featureMap.put("actionstatic_cvr", RankExtractorFeature_20240530.calDiv(
+                midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
+                midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
+              ))
+            }
+
+            val e1: JSONObject = if (record.isNull("e1_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("e1_feature"))
+            val e2: JSONObject = if (record.isNull("e2_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("e2_feature"))
+            val title = b1.getOrDefault("cidtitle", "").toString
+            if (title.nonEmpty) {
+              for ((en, prefix1) <- List((e1, "e1"), (e2, "e2"))) {
+                for (prefix2 <- List("tags_3d", "tags_7d", "tags_14d")) {
+                  if (en.nonEmpty && en.containsKey(prefix2) && en.getString(prefix2).nonEmpty) {
+                    val (f1, f2, f3, f4) = funcC34567ForTags(en.getString(prefix2), title)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_matchnum", f1)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_maxscore", f3)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_avgscore", f4)
+
+                  }
+                }
+              }
+            }
+
+            val d1: JSONObject = if (record.isNull("d1_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("d1_feature"))
+            val d2: JSONObject = if (record.isNull("d2_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("d2_feature"))
+            val d3: JSONObject = if (record.isNull("d3_feature")) new JSONObject() else
+              JSON.parseObject(record.getString("d3_feature"))
+
+            if (d1.nonEmpty) {
+              for (prefix <- List("3h", "6h", "12h", "1d", "3d", "7d")) {
+                val view = if (!d1.containsKey("ad_view_" + prefix)) 0D else d1.getIntValue("ad_view_" + prefix).toDouble
+                val click = if (!d1.containsKey("ad_click_" + prefix)) 0D else d1.getIntValue("ad_click_" + prefix).toDouble
+                val conver = if (!d1.containsKey("ad_conversion_" + prefix)) 0D else d1.getIntValue("ad_conversion_" + prefix).toDouble
+                val income = if (!d1.containsKey("ad_income_" + prefix)) 0D else d1.getIntValue("ad_income_" + prefix).toDouble
+                val cpc = if (click == 0) 0D else income / click
+                val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                val f4 = conver
+                val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                featureMap.put("d1_feature" + "_" + prefix + "_" + "ctr", f1)
+                featureMap.put("d1_feature" + "_" + prefix + "_" + "ctcvr", f2)
+                featureMap.put("d1_feature" + "_" + prefix + "_" + "cvr", f3)
+                featureMap.put("d1_feature" + "_" + prefix + "_" + "conver", f4)
+                featureMap.put("d1_feature" + "_" + prefix + "_" + "ecpm", f5)
+              }
+            }
+
+            val vidRankMaps = scala.collection.mutable.Map[String, scala.collection.immutable.Map[String, Double]]()
+            if (d2.nonEmpty) {
+              d2.foreach(r => {
+                val key = r._1
+                val value = d2.getString(key).split(",").map(r => {
+                  val rList = r.split(":")
+                  (rList(0), rList(2).toDouble)
+                }).toMap
+                vidRankMaps.put(key, value)
+              })
+            }
+            for (prefix1 <- List("ctr", "ctcvr", "ecpm")) {
+              for (prefix2 <- List("1d", "3d", "7d", "14d")) {
+                if (vidRankMaps.contains(prefix1 + "_" + prefix2)) {
+                  val rank = vidRankMaps(prefix1 + "_" + prefix2).getOrDefault(cid, 0.0)
+                  if (rank >= 1.0) {
+                    featureMap.put("vid_rank_" + prefix1 + "_" + prefix2, 1.0 / rank)
+                  }
+                }
+              }
+            }
+
+            if (d3.nonEmpty) {
+              val vTitle = d3.getString("title")
+              val score = Similarity.conceptSimilarity(title, vTitle)
+              featureMap.put("ctitle_vtitle_similarity", score);
+            }
+
+            // k1~k4 事件特征：view/click 来自 ad_*，conver 按 targeting_conversion 取对应事件计数
+            val targetingConversion = Option(record.getString("targeting_conversion")).getOrElse("")
+            val kList: List[(String, JSONObject)] = List(
+              ("k1", k1), ("k2", k2), ("k3", k3), ("k4", k4)
+            )
+            val kPeriods = List("2h", "4h", "6h", "12h", "1d", "3d", "today", "1w")
+            for ((kPrefix, kn) <- kList) {
+              for (period <- kPeriods) {
+                val view = if (kn.isEmpty) 0D else kn.getIntValue("ad_view_" + period).toDouble
+                val click = if (kn.isEmpty) 0D else kn.getIntValue("ad_click_" + period).toDouble
+                val eventJson = parseEventJson(kn, "event_" + period)
+                val conver =
+                  if (eventJson.isEmpty || targetingConversion.isEmpty) 0D
+                  else eventJson.getIntValue(targetingConversion + "_" + period).toDouble
+                val ctr = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                val cvr = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                val ctvr = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                val featPrefix = kPrefix + "_" + period
+                featureMap.put(featPrefix + "_view", view)
+                featureMap.put(featPrefix + "_click", click)
+                featureMap.put(featPrefix + "_conver", conver)
+                featureMap.put(featPrefix + "_ctr", ctr)
+                featureMap.put(featPrefix + "_cvr", cvr)
+                featureMap.put(featPrefix + "_ctvr", ctvr)
+              }
+            }
+
+            /*
+            广告
+              sparse：cid adid adverid targeting_conversion
+
+              cpa --> 1个
+              adverid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr conver ecpm  --> 30个
+              cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              地理//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              app//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              手机品牌//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              系统 无数据
+              week//cid下的 7d 14d、 ctr ctcvr cvr ecpm conver --> 10个
+              hour//cid下的 7d 14d、 ctr ctcvr cvr ecpm conver --> 10个
+
+            用户
+              用户历史 点击/转化 的title tag；3d 7d 14d； cid的title； 数量/最高分/平均分 --> 18个
+              用户历史 14d 看过/点过/转化次数/income； ctr cvr ctcvr ecpm；  --> 8个
+
+              用户到cid的ui特征 --> 10个
+                1/用户最近看过这个cid的时间间隔
+                1/用户最近点过这个cid的时间间隔
+                1/用户最近转过这个cid的时间间隔
+                用户看过这个cid多少次
+                用户点过这个cid多少次
+                用户转过这个cid多少次
+                用户对这个cid花了多少钱
+                用户对这个cid的ctr ctcvr cvr
+
+            视频
+              title与cid的 sim-score-1/-2 无数据
+              vid//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              vid//cid下的 1d 3d 7d 14d、 ctr ctcvr ecpm 的rank值 倒数 --> 12个
+
+             */
+
+
+            //4 处理label信息（v3 表 label 在 label_json 内）。
+            val labels = new JSONObject
+            val labelJson = getJsonObject(record, "label_json")
+            for (labelKey <- List("ad_is_click", "ad_is_conversion")) {
+              if (labelJson.containsKey(labelKey)) {
+                labels.put(labelKey, labelJson.getString(labelKey))
+              }
+            }
+            //5 处理log key表头。
+            val mid = record.getString("mid")
+            val headvideoid = record.getString("headvideoid")
+            val logKey = (apptype, mid, cid, ts, headvideoid).productIterator.mkString(",")
+            val labelKey = labels.toString()
+            val featureKey = featureMap.toString()
+            //6 拼接数据，保存。
+            logKey + "\t" + labelKey + "\t" + featureKey
+          })
+
+        // 4 保存数据到hdfs
+        val savePartition = dt + hh
+        val hdfsPath = savePath + "/" + savePartition
+        if (hdfsPath.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
+          println("删除路径并开始数据写入:" + hdfsPath)
+          MyHdfsUtils.delete_hdfs_path(hdfsPath)
+          odpsData.coalesce(repartition).saveAsTextFile(hdfsPath, classOf[GzipCodec])
+        } else {
+          println("路径不合法，无法写入:" + hdfsPath)
+        }
+      }
+
+    }
+  }
+
+  def func(record: Record, schema: TableSchema): Record = {
+    record
+  }
+
+  def funcC34567ForTags(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
+    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
+    val tagsList = tags.split(",")
+    var d1 = 0.0
+    val d2 = new ArrayBuffer[String]()
+    var d3 = 0.0
+    var d4 = 0.0
+    for (tag <- tagsList) {
+      if (title.contains(tag)) {
+        d1 = d1 + 1.0
+        d2.add(tag)
+      }
+      val score = Similarity.conceptSimilarity(tag, title)
+      d3 = if (score > d3) score else d3
+      d4 = d4 + score
+    }
+    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
+    (d1, d2.mkString(","), d3, d4)
+  }
+
+  def getJsonObject(record: Record, name: String): JSONObject = {
+    if (record.isNull(name)) {
+      new JSONObject()
+    } else {
+      JSON.parseObject(record.getString(name))
+    }
+  }
+
+  def parseEventJson(feature: JSONObject, eventKey: String): JSONObject = {
+    if (feature == null || feature.isEmpty || !feature.containsKey(eventKey)) {
+      return new JSONObject()
+    }
+    val raw = feature.getString(eventKey)
+    if (raw == null || raw.isEmpty) {
+      return new JSONObject()
+    }
+    try {
+      val parsed = JSON.parseObject(raw)
+      if (parsed == null) new JSONObject() else parsed
+    } catch {
+      case _: Exception => new JSONObject()
+    }
+  }
+}

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_32_bucket_20260807.scala

@@ -0,0 +1,153 @@
+package com.aliyun.odps.spark.examples.makedata_ad.v20240718
+
+import com.alibaba.fastjson.JSON
+import com.aliyun.odps.spark.examples.myUtils.{MyHdfsUtils, ParamUtils}
+import org.apache.hadoop.io.compress.GzipCodec
+import org.apache.spark.Partitioner
+import org.apache.spark.sql.SparkSession
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
+import scala.io.Source
+
+/**
+ * 20260807 分桶边界生成：
+ * 1) 基于 origin 样本计算 k1~k4 事件特征分桶
+ * 2) 与已有分桶文件（默认 20260410_ad_bucket_920.txt）合并后写出
+ */
+object makedata_ad_32_bucket_20260807 {
+  class FeaturePartitioner(featureNames: Map[String, Int]) extends Partitioner {
+    override def numPartitions: Int = featureNames.size
+
+    override def getPartition(key: Any): Int = {
+      val featureName = key.asInstanceOf[String]
+      featureNames.getOrElse(featureName, 0)
+    }
+  }
+
+  def main(args: Array[String]): Unit = {
+
+    val spark = SparkSession
+      .builder()
+      .appName(this.getClass.getName)
+      .getOrCreate()
+    val sc = spark.sparkContext
+
+    // 1 读取参数
+    val param = ParamUtils.parseArgs(args)
+    val readPath = param.getOrElse("readPath", "/dw/recommend/model/31_ad_sample_data/20240620*")
+    val savePath = param.getOrElse("savePath", "/dw/recommend/model/32_bucket_data/")
+    val fileName = param.getOrElse("fileName", "20260807_ad_bucket_1112")
+    val sampleRate = param.getOrElse("sampleRate", "1.0").toDouble
+    val bucketNum = param.getOrElse("bucketNum", "100").toInt
+    // 仅计算这些特征的分桶；默认自动生成全部 k 特征名。也可传 featureNameFile 覆盖
+    val featureNameFile = param.getOrElse("featureNameFile", "20260807_ad_feature_name.txt")
+    // 已有分桶文件（resources），与新计算的 k 分桶合并
+    val existingBucketFile = param.getOrElse("existingBucketFile", "20260410_ad_bucket_920.txt")
+
+    val loader = getClass.getClassLoader
+    val featureNames =
+      if (featureNameFile.nonEmpty) {
+        loadResourceLines(loader, featureNameFile)
+          .map(_.replace(" ", ""))
+          .filter(_.nonEmpty)
+          .distinct
+          .toList
+      } else {
+        buildKFeatureNames()
+      }
+    println(s"featureNameFile=${if (featureNameFile.isEmpty) "<auto:k1~k4>" else featureNameFile}")
+    println(s"featureNames.size=${featureNames.size}")
+    val featureNamesSet = featureNames.toSet
+
+    val existingBucketLines = loadResourceLines(loader, existingBucketFile)
+      .filter(_.nonEmpty)
+      .filter(line => !featureNamesSet.contains(line.split("\t")(0)))
+    println(s"existingBucketFile=$existingBucketFile keepLines=${existingBucketLines.size}")
+
+    val data = sc.textFile(readPath)
+
+    // 2 读取特征数据、打平后分区
+    val flattenData = data
+      .sample(false, sampleRate)
+      .flatMap(r => {
+        val rList = r.split("\t")
+        val jsons = JSON.parseObject(rList(2))
+        jsons.map(r => (r._1, jsons.getDoubleValue(r._1)))
+      }).filter(r => r._2 > 1E-8)
+      .filter(r => featureNamesSet.contains(r._1))
+      .partitionBy(new FeaturePartitioner(featureNames.zipWithIndex.toMap))
+
+    // 3 计算分桶值
+    val newBucketRdd = flattenData.mapPartitions(iter => {
+      if (iter.isEmpty) {
+        Array[String]().iterator
+      } else {
+        val headValue = iter.next()
+        val key = headValue._1
+        val leftValues = iter.map(_._2).toArray
+        val sortedValues = (Array(headValue._2) ++ leftValues).sorted
+        val len = sortedValues.length
+        val oneBucketNum = (len - 1) / (bucketNum - 1) + 1 // 确保每个桶至少有一个元素
+        val buffers = new ArrayBuffer[Double]()
+
+        var lastBucketValue = sortedValues(0) // 记录上一个桶的切分点
+        for (j <- 0 until len by oneBucketNum) {
+          val d = sortedValues(j)
+          if (j > 0 && d != lastBucketValue) {
+            // 如果当前切分点不同于上一个切分点，则保存当前切分点
+            buffers += d
+          }
+          lastBucketValue = d // 更新上一个桶的切分点
+        }
+
+        // 最后一个桶的结束点应该是数组的最后一个元素
+        if (!buffers.contains(sortedValues.last)) {
+          buffers += sortedValues.last
+        }
+        Array(key + "\t" + bucketNum.toString + "\t" + buffers.mkString(",")).iterator
+      }
+    })
+
+    val existingBucketRdd = sc.parallelize(existingBucketLines, 1)
+    val resultRdd = existingBucketRdd.union(newBucketRdd)
+
+    // 4 保存数据到hdfs
+    val hdfsPath = savePath + "/" + fileName
+    if (hdfsPath.nonEmpty && fileName.nonEmpty && hdfsPath.startsWith("/dw/recommend/model/")) {
+      println("删除路径并开始数据写入:" + hdfsPath)
+      MyHdfsUtils.delete_hdfs_path(hdfsPath)
+      resultRdd.repartition(1).saveAsTextFile(hdfsPath, classOf[GzipCodec])
+    } else {
+      println("路径不合法，无法写入:" + hdfsPath)
+    }
+  }
+
+  def buildKFeatureNames(): List[String] = {
+    val kPrefixes = List("k1", "k2", "k3", "k4")
+    val kPeriods = List("2h", "4h", "6h", "12h", "1d", "3d", "today", "1w")
+    val kMetrics = List("view", "click", "conver", "ctr", "cvr", "ctvr")
+    for {
+      kPrefix <- kPrefixes
+      period <- kPeriods
+      metric <- kMetrics
+    } yield s"${kPrefix}_${period}_${metric}"
+  }
+
+  def loadResourceLines(loader: ClassLoader, fileName: String): Array[String] = {
+    if (fileName == null || fileName.isEmpty) {
+      return Array.empty[String]
+    }
+    val resourceUrl = loader.getResource(fileName)
+    if (resourceUrl == null) {
+      println(s"existing bucket file not found in resources: $fileName")
+      return Array.empty[String]
+    }
+    val source = Source.fromURL(resourceUrl)
+    try {
+      source.getLines().map(_.replace(" ", "")).filter(_.nonEmpty).toArray
+    } finally {
+      source.close()
+    }
+  }
+}

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20260808.scala

@@ -0,0 +1,901 @@
+package com.aliyun.odps.spark.examples.makedata_ad.v20240718
+
+import com.alibaba.fastjson.{JSON, JSONObject}
+import com.aliyun.odps.TableSchema
+import com.aliyun.odps.data.Record
+import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, ParamUtils, env}
+import examples.extractor.{ExtractorUtils, RankExtractorFeature_20240530}
+import examples.utils.{AdUtil, DateTimeUtil, SimilarityUtils}
+import org.apache.spark.sql.SparkSession
+
+import java.time.{Instant, ZoneId, ZonedDateTime}
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
+import scala.io.Source
+import scala.language.postfixOps
+import scala.util.Random
+
+object makedata_ad_33_bucketDataFromOriginToHive_20260808 {
+  val CTR_SMOOTH_BETA_FACTOR = 25
+  val CVR_SMOOTH_BETA_FACTOR = 10
+  val CTCVR_SMOOTH_BETA_FACTOR = 100
+
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession
+      .builder()
+      .appName(this.getClass.getName)
+      .getOrCreate()
+    val sc = spark.sparkContext
+
+
+    // 1 读取参数
+    val param = ParamUtils.parseArgs(args)
+    val tablePart = param.getOrElse("tablePart", "64").toInt
+    val beginStr = param.getOrElse("beginStr", "20250216")
+    val endStr = param.getOrElse("endStr", "20250216")
+    val project = param.getOrElse("project", "loghubods")
+    val inputTable = param.getOrElse("inputTable", "alg_recsys_ad_sample_all")
+    val outputTable = param.getOrElse("outputTable", "ad_easyrec_train_data_v1_sampled")
+    val outputTable2 = param.getOrElse("outputTable2", "")
+    val filterHours = param.getOrElse("filterHours", "00,01,02,03,04,05,06,07").split(",").toSet
+    val idDefaultValue = param.getOrElse("idDefaultValue", "1.0").toDouble
+    val filterNames = param.getOrElse("filterNames", "").split(",").filter(_.nonEmpty).toSet
+    val filterAdverIds = param.getOrElse("filterAdverIds", "").split(",").filter(_.nonEmpty).toSet
+    val whatLabel = param.getOrElse("whatLabel", "ad_is_conversion")
+    val negSampleRate = param.getOrElse("negSampleRate", "1").toDouble
+    // 分割样本集的比例，splitRate部分输出至outputTable，补集输出至outputTable2（如果outputTable2不为空）
+    val splitRate = param.getOrElse("splitRate", "0.9").toDouble
+    val maskFeatureRate = param.getOrElse("maskFeatureRate", "0.0").toDouble
+    val bucketFile = param.getOrElse("bucketFile", "20260807_ad_bucket_1112.txt")
+    val flag = param.getOrElse("flag", "0")
+
+    val loader = getClass.getClassLoader
+    val resourceUrlBucket = loader.getResource(bucketFile)
+    val buckets =
+      if (resourceUrlBucket != null) {
+        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
+        Source.fromURL(resourceUrlBucket).close()
+        buckets
+      } else {
+        ""
+      }
+    val bucketsMap = buckets.split("\n")
+      .map(r => r.replace(" ", "").replaceAll("\n", ""))
+      .filter(r => r.nonEmpty)
+      .map(r => {
+        val rList = r.split("\t")
+        val featureName = rList(0).replace("*", "_x_").replace("(view)", "_view")
+        (featureName, (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
+      }).toMap
+    println(bucketsMap.keySet)
+    val bucketsMap_br = sc.broadcast(bucketsMap)
+    val denseFeatureNames = bucketsMap.keySet
+    val lowerCaseDenseFeatureNames = bucketsMap.keySet.map(_.toLowerCase)
+    val sparseFeatureNames = Set(
+      "cid", "adid", "adverid", "targeting_conversion",
+      "region", "city", "brand",
+      "vid", "cate1", "cate2",
+      "user_cid_click_list", "user_cid_conver_list",
+      "user_vid_return_tags_2h", "user_vid_return_tags_1d", "user_vid_return_tags_3d", "user_vid_return_tags_7d",
+      "user_vid_return_tags_14d", "apptype", "ts", "mid", "pqtid", "hour", "hour_quarter", "root_source_scene",
+      "root_source_channel", "is_first_layer", "title_split", "profession", "user_vid_share_tags_1d", "user_vid_share_tags_14d",
+      "user_vid_return_cate1_14d", "user_vid_return_cate2_14d", "user_vid_share_cate1_14d", "user_vid_share_cate2_14d",
+      "creative_type", "creative_hook_embedding", "creative_why_embedding", "creative_action_embedding", "user_has_conver_1y",
+      "user_adverid_view_3d", "user_adverid_view_7d", "user_adverid_view_30d",
+      "user_adverid_click_3d", "user_adverid_click_7d", "user_adverid_click_30d",
+      "user_adverid_conver_3d", "user_adverid_conver_7d", "user_adverid_conver_30d",
+      "user_skuid_view_3d", "user_skuid_view_7d", "user_skuid_view_30d",
+      "user_skuid_click_3d", "user_skuid_click_7d", "user_skuid_click_30d",
+      "user_skuid_conver_3d", "user_skuid_conver_7d", "user_skuid_conver_30d",
+      "is_weekday", "day_of_the_week", "user_conver_ad_class", "category_name",
+      "material_md5", "user_layer", "user_layer_l6", "user_class", "user_click_ad_class", "user_view_ad_class",
+      "customer", "customer_id", "landing", "landing_page_type", "agent_id", "flag")
+
+
+    // 2 读取odps+表信息
+    val odpsOps = env.getODPS(sc)
+
+    val tableSchema = odpsOps.getTableSchema(project, outputTable, isPartition = false)
+
+    // 检查所有字段，收集非法字段
+    val invalidFields = tableSchema.flatMap { case (fieldName, _) =>
+      // 跳过 has_click 和 has_conversion 列
+      if (fieldName != "has_click" && fieldName != "has_conversion") {
+        if (!lowerCaseDenseFeatureNames.contains(fieldName) && !sparseFeatureNames.contains(fieldName)) {
+          Some(fieldName) // 收集缺少字段
+        } else {
+          None
+        }
+      } else {
+        None
+      }
+    }.toList
+
+    // 如果存在非法字段，抛出标准异常
+    if (invalidFields.nonEmpty) {
+      throw new IllegalArgumentException(s"缺少字段: ${invalidFields.mkString(", ")}")
+    }
+
+    // 3 循环执行数据生产
+    val dateRange = MyDateUtils.getDateRange(beginStr, endStr)
+    for (dt <- dateRange) {
+      val timeRange = MyDateUtils.getDateHourRange(dt + "06", dt + "23")
+      val recordRdd = timeRange.map { dt_hh =>
+          val dt = dt_hh.substring(0, 8)
+          val hh = dt_hh.substring(8, 10)
+          val partition = s"dt=$dt,hh=$hh"
+          if (filterHours.nonEmpty && filterHours.contains(hh)) {
+            None
+          } else {
+            Some(partition)
+          }
+        }.collect {
+          case Some(partition) => partition
+        }.map(partition => {
+          val odpsData = odpsOps.readTable(project = project,
+              table = inputTable,
+              partition = partition,
+              transfer = func,
+              numPartition = tablePart)
+            .filter(record => {
+              AdUtil.isApi(record)
+            })
+            .filter(record => {
+              val extendAlg = Option(record.getString("extend_alg"))
+                .filter(_.nonEmpty)
+                .map(JSON.parseObject)
+                .getOrElse(new JSONObject())
+              Option(extendAlg.getString("extractstrategy")).contains("engine")
+            })
+            .filter(record => {
+              val appType = record.getString("apptype")
+              !Set("12", "13").contains(appType)
+            })
+            .filter(record => {
+              val adverId = record.getString("adverid")
+              !filterAdverIds.contains(adverId)
+            })
+            .filter(record => {
+              val labelJson = getJsonObject(record, "label_json")
+              val label = labelJson.getIntValue(whatLabel)
+              label > 0 || Random.nextDouble() < negSampleRate
+            })
+            .map(record => {
+              val featureMap = new JSONObject()
+              val ts = record.getString("ts").toInt
+              val instant = Instant.ofEpochSecond(ts)
+              // 设置时区为中国时区
+              val chinaZone = ZoneId.of("Asia/Shanghai")
+              // 将 Instant 对象转换为中国时区的 ZonedDateTime 对象
+              val zonedDateTime = ZonedDateTime.ofInstant(instant, chinaZone)
+              // 获取星期几（1=周一，7=周日）
+              val dayOfTheWeek = zonedDateTime.getDayOfWeek.getValue()
+              val isWeekday = if (dayOfTheWeek <= 5) 1 else 2
+              val cid = record.getString("cid")
+              val mid = record.getString("mid")
+              val pqtid = record.getString("pqtid")
+              val apptype = record.getString("apptype")
+              val targetingConversion = Option(record.getString("targeting_conversion")).getOrElse("")
+
+              featureMap.put("apptype", apptype)
+              featureMap.put("ts", ts)
+              featureMap.put("mid", mid)
+              featureMap.put("pqtid", pqtid)
+              featureMap.put("targeting_conversion", targetingConversion)
+              val extend: JSONObject = if (record.isNull("extend")) new JSONObject() else
+                JSON.parseObject(record.getString("extend"))
+              val extendAlg: JSONObject = getJsonObject(record, "extend_alg")
+              val mateFeature: JSONObject = if (record.isNull("metafeaturemap")) new JSONObject() else
+                JSON.parseObject(record.getString("metafeaturemap"))
+              val reqFeature: JSONObject = if (!mateFeature.containsKey("reqFeature")) new JSONObject() else
+                mateFeature.getJSONObject("reqFeature")
+              val sceneFeature: JSONObject = if (!mateFeature.containsKey("sceneFeature")) new JSONObject() else
+                mateFeature.getJSONObject("sceneFeature")
+              val b1: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_basic_info")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_basic_info")
+              val b2: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_adver_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_adver_action")
+              val b3: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_cid_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_cid_action")
+              val b4: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_region_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_region_action")
+              val b5: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_app_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_app_action")
+              val b6: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_week_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_week_action")
+              val b7: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_hour_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_hour_action")
+              val b8: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_brand_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_brand_action")
+              val b9: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_weChatVersion_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_weChatVersion_action")
+              val j1: JSONObject = getJsonObject(record, "j1_feature") // user_layer
+              val j2: JSONObject = getJsonObject(record, "j2_feature") // user_layer x advertiser
+              val j3: JSONObject = getJsonObject(record, "j3_feature") // user_layer x customer
+              val j4: JSONObject = getJsonObject(record, "j4_feature") // user_layer x profession
+              val j5: JSONObject = getJsonObject(record, "j5_feature") // user_layer x category
+              val j6: JSONObject = getJsonObject(record, "j6_feature") // user_layer x cid
+              val j7: JSONObject = getJsonObject(record, "j7_feature") // landingpage
+              val j8: JSONObject = getJsonObject(record, "j8_feature") // landingpage x advertiser
+              val j9: JSONObject = getJsonObject(record, "j9_feature") // landingpage x customer
+              val j10: JSONObject = getJsonObject(record, "j10_feature") // landingpage x profession
+              val j11: JSONObject = getJsonObject(record, "j11_feature") // landingpage x category
+              val k1: JSONObject = getJsonObject(record, "k1_feature")
+              val k2: JSONObject = getJsonObject(record, "k2_feature")
+              val k3: JSONObject = getJsonObject(record, "k3_feature")
+              val k4: JSONObject = getJsonObject(record, "k4_feature")
+
+              featureMap.put("cid_" + cid, idDefaultValue)
+              if (b1.containsKey("adid") && b1.getString("adid").nonEmpty) {
+                featureMap.put("adid_" + b1.getString("adid"), idDefaultValue)
+              }
+              if (b1.containsKey("adverid") && b1.getString("adverid").nonEmpty) {
+                featureMap.put("adverid_" + b1.getString("adverid"), idDefaultValue)
+              }
+              if (b1.containsKey("targeting_conversion") && b1.getString("targeting_conversion").nonEmpty) {
+                featureMap.put("targeting_conversion_" + b1.getString("targeting_conversion"), idDefaultValue)
+              }
+              if (b1.containsKey("creative_type") && b1.getString("creative_type").nonEmpty) {
+                featureMap.put("creative_type", b1.getString("creative_type"))
+              }
+              if (b1.containsKey("creative_hook_embedding") && b1.getString("creative_hook_embedding").nonEmpty) {
+                featureMap.put("creative_hook_embedding", b1.getString("creative_hook_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
+              }
+              if (b1.containsKey("creative_why_embedding") && b1.getString("creative_why_embedding").nonEmpty) {
+                featureMap.put("creative_why_embedding", b1.getString("creative_why_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
+              }
+              if (b1.containsKey("creative_action_embedding") && b1.getString("creative_action_embedding").nonEmpty) {
+                featureMap.put("creative_action_embedding", b1.getString("creative_action_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
+              }
+              if (extendAlg.containsKey("customer_id") && extendAlg.getString("customer_id").nonEmpty) {
+                featureMap.put("customer", extendAlg.getString("customer_id"))
+                featureMap.put("customer_id", extendAlg.getString("customer_id"))
+              }
+              if (sceneFeature.containsKey("hour") && sceneFeature.getString("hour").nonEmpty) {
+                featureMap.put("hour", sceneFeature.getString("hour"))
+              }
+              if (sceneFeature.containsKey("hour_quarter") && sceneFeature.getString("hour_quarter").nonEmpty) {
+                featureMap.put("hour_quarter", sceneFeature.getString("hour_quarter"))
+              }
+              featureMap.put("is_weekday", isWeekday)
+              featureMap.put("day_of_the_week", dayOfTheWeek)
+
+              val hour = DateTimeUtil.getHourByTimestamp(ts)
+              featureMap.put("hour_" + hour, idDefaultValue)
+
+              val dayOfWeek = DateTimeUtil.getDayOrWeekByTimestamp(ts)
+              featureMap.put("dayofweek_" + dayOfWeek, idDefaultValue);
+
+              featureMap.put("apptype_" + apptype, idDefaultValue);
+
+              if (extend.containsKey("abcode") && extend.getString("abcode").nonEmpty) {
+                featureMap.put("abcode_" + extend.getString("abcode"), idDefaultValue)
+              }
+
+              // 定义需要处理的键名列表
+              val reqFeatureKeys = List(
+                "cid", "adid", "adverid", "profession", "region",
+                "city", "is_first_layer", "root_source_scene",
+                "root_source_channel", "brand", "vid", "category_name", "material_md5"
+              )
+
+              // 使用函数式方式处理所有键
+              reqFeatureKeys.foreach { key =>
+                val value = reqFeature.getString(key)
+
+                // 检查值是否非空
+                if (value != null && value.nonEmpty) {
+                  featureMap.put(key, value)
+                }
+              }
+              if (extendAlg.containsKey("landing_page_type") && extendAlg.getString("landing_page_type").nonEmpty) {
+                featureMap.put("landing", extendAlg.getString("landing_page_type"))
+                featureMap.put("landing_page_type", extendAlg.getString("landing_page_type"))
+              }
+              if (reqFeature.containsKey("agentId") && reqFeature.getString("agentId").nonEmpty) {
+                featureMap.put("agent_id", reqFeature.getString("agentId"))
+              }
+              if (reqFeature.containsKey("layer_l4")) {
+                featureMap.put("user_layer", reqFeature.getString("layer_l4"))
+              }
+              if (reqFeature.containsKey("layer_l6") && reqFeature.getString("layer_l6").nonEmpty) {
+                featureMap.put("user_layer_l6", reqFeature.getString("layer_l6"))
+              } else {
+                featureMap.put("user_layer_l6", "无曝光")
+              }
+              if (reqFeature.containsKey("clazz_l4")) {
+                featureMap.put("user_class", reqFeature.getString("clazz_l4"))
+              }
+              if (b1.containsKey("cpa")) {
+                featureMap.put("cpa", b1.getString("cpa").toDouble)
+              }
+              if (b1.containsKey("weight") && b1.getString("weight").nonEmpty) {
+                featureMap.put("weight", b1.getString("weight").toDouble)
+              }
+
+              for ((bn, prefix1) <- List(
+                (b2, "b2"), (b3, "b3"), (b4, "b4"), (b5, "b5"), (b8, "b8"), (b9, "b9")
+              )) {
+                for (prefix2 <- List(
+                  "3h", "6h", "12h", "1d", "3d", "7d", "today", "yesterday"
+                )) {
+                  val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                  val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                  val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                  val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                  // NOTE(zhoutian):
+                  // 这里cpc只是为了计算cpm的平滑的工具量，没有实际业务意义，因为cpm并非比率，本身不适合直接计算Wilson平滑
+                  // 不使用cpa的原因是未来可能出现广告采用cpc计费的情况或者无法获取转化量的情况，用点击更为稳定
+                  // 其它几组特征亦采用相同逻辑
+                  // 2025-02-17改为增加固定分母平滑，income实际已经可以直接参与cpm平滑计算
+                  val cpc = if (click == 0) 0D else income / click
+                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                  val f4 = conver
+                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
+                }
+              }
+
+              for ((bn, prefix1) <- List(
+                (b6, "b6"), (b7, "b7")
+              )) {
+                for (prefix2 <- List(
+                  "7d", "14d"
+                )) {
+                  val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                  val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                  val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                  val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                  val cpc = if (click == 0) 0D else income / click
+                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                  val f4 = conver
+                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
+                }
+              }
+
+
+              val c1: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_ad_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_mid_feature_ad_action")
+
+              val midActionList = if (c1.containsKey("action") && c1.getString("action").nonEmpty) {
+                c1.getString("action").split(",").map(r => {
+                  val rList = r.split(":")
+                  (rList(0), (rList(1).toInt, rList(2).toInt, rList(3).toInt, rList(4).toInt, rList(5)))
+                }).sortBy(-_._2._1).toList
+              } else {
+                new ArrayBuffer[(String, (Int, Int, Int, Int, String))]().toList
+              }
+              // u特征
+              val viewAll = midActionList.size.toDouble
+              val clickAll = midActionList.map(_._2._2).sum.toDouble
+              val converAll = midActionList.map(_._2._3).sum.toDouble
+              val incomeAll = midActionList.map(_._2._4).sum.toDouble
+              featureMap.put("viewAll", viewAll)
+              featureMap.put("clickAll", clickAll)
+              featureMap.put("converAll", converAll)
+              featureMap.put("incomeAll", incomeAll)
+              featureMap.put("ctr_all", RankExtractorFeature_20240530.calDiv(clickAll, viewAll))
+              featureMap.put("ctcvr_all", RankExtractorFeature_20240530.calDiv(converAll, viewAll))
+              featureMap.put("cvr_all", RankExtractorFeature_20240530.calDiv(clickAll, converAll))
+              featureMap.put("ecpm_all", RankExtractorFeature_20240530.calDiv(incomeAll * 1000, viewAll))
+
+              if (c1.containsKey("user_has_conver_1y") && c1.getInteger("user_has_conver_1y") != null) {
+                featureMap.put("user_has_conver_1y", c1.getInteger("user_has_conver_1y"))
+              }
+              if (c1.containsKey("user_conver_ad_class") && c1.getString("user_conver_ad_class") != null) {
+                featureMap.put("user_conver_ad_class", c1.getString("user_conver_ad_class"))
+              }
+              if (c1.containsKey("user_click_ad_class") && c1.getString("user_click_ad_class") != null) {
+                featureMap.put("user_click_ad_class", c1.getString("user_click_ad_class"))
+              }
+              if (c1.containsKey("user_view_ad_class") && c1.getString("user_view_ad_class") != null) {
+                featureMap.put("user_view_ad_class", c1.getString("user_view_ad_class"))
+              }
+
+              // ui特征
+              val midTimeDiff = scala.collection.mutable.Map[String, Double]()
+              midActionList.foreach {
+                case (cid, (ts_history, click, conver, income, title)) =>
+                  if (!midTimeDiff.contains("timediff_view_" + cid)) {
+                    midTimeDiff.put("timediff_view_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                  }
+                  if (!midTimeDiff.contains("timediff_click_" + cid) && click > 0) {
+                    midTimeDiff.put("timediff_click_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                  }
+                  if (!midTimeDiff.contains("timediff_conver_" + cid) && conver > 0) {
+                    midTimeDiff.put("timediff_conver_" + cid, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                  }
+              }
+
+              val midActionStatic = scala.collection.mutable.Map[String, Double]()
+              midActionList.foreach {
+                case (cid, (ts_history, click, conver, income, title)) =>
+                  midActionStatic.put("actionstatic_view_" + cid, 1.0 + midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0))
+                  midActionStatic.put("actionstatic_click_" + cid, click + midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0))
+                  midActionStatic.put("actionstatic_conver_" + cid, conver + midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0))
+                  midActionStatic.put("actionstatic_income_" + cid, income + midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0))
+              }
+
+              val clickCidList = collection.mutable.ListBuffer[String]()
+              val converCidList = collection.mutable.ListBuffer[String]()
+              midActionList.foreach {
+                case (cid, (ts_history, click, conver, income, title)) =>
+                  if (click == 1) clickCidList += cid
+                  if (conver == 1) converCidList += cid
+              }
+              if (clickCidList.nonEmpty) {
+                featureMap.put("user_cid_click_list", clickCidList.takeRight(50).mkString(","))
+              } else {
+                featureMap.put("user_cid_click_list", "")
+              }
+              if (converCidList.nonEmpty) {
+                featureMap.put("user_cid_conver_list", converCidList.takeRight(50).mkString(","))
+              } else {
+                featureMap.put("user_cid_conver_list", "")
+              }
+              if (midTimeDiff.contains("timediff_view_" + cid)) {
+                featureMap.put("timediff_view", midTimeDiff.getOrDefault("timediff_view_" + cid, 0.0))
+              }
+              if (midTimeDiff.contains("timediff_click_" + cid)) {
+                featureMap.put("timediff_click", midTimeDiff.getOrDefault("timediff_click_" + cid, 0.0))
+              }
+              if (midTimeDiff.contains("timediff_conver_" + cid)) {
+                featureMap.put("timediff_conver", midTimeDiff.getOrDefault("timediff_conver_" + cid, 0.0))
+              }
+              if (midActionStatic.contains("actionstatic_view_" + cid)) {
+                featureMap.put("actionstatic_view", midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0))
+              }
+              if (midActionStatic.contains("actionstatic_click_" + cid)) {
+                featureMap.put("actionstatic_click", midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0))
+              }
+              if (midActionStatic.contains("actionstatic_conver_" + cid)) {
+                featureMap.put("actionstatic_conver", midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0))
+              }
+              if (midActionStatic.contains("actionstatic_income_" + cid)) {
+                featureMap.put("actionstatic_income", midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0))
+              }
+              if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
+                featureMap.put("actionstatic_ctr", RankExtractorFeature_20240530.calDiv(
+                  midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                  midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+                ))
+              }
+              if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_conver_" + cid)) {
+                featureMap.put("actionstatic_ctcvr", RankExtractorFeature_20240530.calDiv(
+                  midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
+                  midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+                ))
+              }
+              if (midActionStatic.contains("actionstatic_conver_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
+                featureMap.put("actionstatic_cvr", RankExtractorFeature_20240530.calDiv(
+                  midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
+                  midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
+                ))
+              }
+
+              val e1: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_return_tags")) new JSONObject() else
+                mateFeature.getJSONObject("alg_mid_feature_return_tags")
+              val e2: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_share_tags")) new JSONObject() else
+                mateFeature.getJSONObject("alg_mid_feature_share_tags")
+              val title = b1.getOrDefault("cidtitle", "").toString
+              if (title.nonEmpty) {
+                for ((en, prefix1) <- List((e1, "e1"), (e2, "e2"))) {
+                  for (prefix2 <- List("tags_3d", "tags_7d", "tags_14d")) {
+                    if (en.nonEmpty && en.containsKey(prefix2) && en.getString(prefix2).nonEmpty) {
+                      val (f1, f2, f3, f4) = funcC34567ForTagsNew(en.getString(prefix2), title)
+                      featureMap.put(prefix1 + "_" + prefix2 + "_matchnum", f1)
+                      featureMap.put(prefix1 + "_" + prefix2 + "_maxscore", f3)
+                      featureMap.put(prefix1 + "_" + prefix2 + "_avgscore", f4)
+
+                    }
+                  }
+                }
+              }
+
+              if (e1.containsKey("tags_2h") && e1.getString("tags_2h").nonEmpty) {
+                featureMap.put("user_vid_return_tags_2h", e1.getString("tags_2h"))
+              }
+              if (e1.containsKey("tags_1d") && e1.getString("tags_1d").nonEmpty) {
+                featureMap.put("user_vid_return_tags_1d", e1.getString("tags_1d"))
+              }
+              if (e1.containsKey("tags_3d") && e1.getString("tags_3d").nonEmpty) {
+                featureMap.put("user_vid_return_tags_3d", e1.getString("tags_3d"))
+              }
+              if (e1.containsKey("tags_7d") && e1.getString("tags_7d").nonEmpty) {
+                featureMap.put("user_vid_return_tags_7d", e1.getString("tags_7d"))
+              }
+              if (e1.containsKey("tags_14d") && e1.getString("tags_14d").nonEmpty) {
+                featureMap.put("user_vid_return_tags_14d", e1.getString("tags_14d"))
+              }
+
+              if (e2.containsKey("tags_14d") && e2.getString("tags_14d").nonEmpty) {
+                featureMap.put("user_vid_share_tags_1d", e2.getString("tags_1d"))
+              }
+              if (e2.containsKey("tags_14d") && e2.getString("tags_14d").nonEmpty) {
+                featureMap.put("user_vid_share_tags_14d", e2.getString("tags_14d"))
+              }
+
+              val g1: JSONObject = if (!mateFeature.containsKey("mid_return_video_cate")) new JSONObject() else
+                mateFeature.getJSONObject("mid_return_video_cate")
+              val g2: JSONObject = if (!mateFeature.containsKey("mid_share_video_cate")) new JSONObject() else
+                mateFeature.getJSONObject("mid_share_video_cate")
+              if (g1.containsKey("cate1_14d") && g1.getString("cate1_14d").nonEmpty) {
+                featureMap.put("user_vid_return_cate1_14d", g1.getString("cate1_14d"))
+              }
+              if (g1.containsKey("cate2_14d") && g1.getString("cate2_14d").nonEmpty) {
+                featureMap.put("user_vid_return_cate2_14d", g1.getString("cate2_14d"))
+              }
+              if (g2.containsKey("cate1_14d") && g2.getString("cate1_14d").nonEmpty) {
+                featureMap.put("user_vid_share_cate1_14d", g2.getString("cate1_14d"))
+              }
+              if (g2.containsKey("cate2_14d") && g2.getString("cate2_14d").nonEmpty) {
+                featureMap.put("user_vid_share_cate2_14d", g2.getString("cate2_14d"))
+              }
+
+              val h1: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_adver_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_mid_feature_adver_action")
+              val h2: JSONObject = if (!mateFeature.containsKey("alg_mid_feature_sku_action")) new JSONObject() else
+                mateFeature.getJSONObject("alg_mid_feature_sku_action")
+
+              // 定义时间维度和对应的前缀
+              val timeDimensions = Seq("3d", "7d", "30d")
+              for (dimension <- timeDimensions) {
+                if (h1.containsKey(dimension) && h1.getString(dimension).nonEmpty) {
+                  val action = h1.getString(dimension).split(",")
+                  if (action.length >= 3) {
+                    featureMap.put(s"user_adverid_view_${dimension}", action(0))
+                    featureMap.put(s"user_adverid_click_${dimension}", action(1))
+                    featureMap.put(s"user_adverid_conver_${dimension}", action(2))
+                  }
+                }
+                if (h2.containsKey(dimension) && h2.getString(dimension).nonEmpty) {
+                  val action = h2.getString(dimension).split(",")
+                  if (action.length >= 3) {
+                    featureMap.put(s"user_skuid_view_${dimension}", action(0))
+                    featureMap.put(s"user_skuid_click_${dimension}", action(1))
+                    featureMap.put(s"user_skuid_conver_${dimension}", action(2))
+                  }
+                }
+              }
+
+
+              val d1: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_vid_cf")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_vid_cf")
+              val d2: JSONObject = if (!mateFeature.containsKey("alg_cid_feature_vid_cf_rank")) new JSONObject() else
+                mateFeature.getJSONObject("alg_cid_feature_vid_cf_rank")
+              val d3: JSONObject = if (!mateFeature.containsKey("alg_vid_feature_basic_info")) new JSONObject() else
+                mateFeature.getJSONObject("alg_vid_feature_basic_info")
+
+              if (d1.nonEmpty) {
+                for (prefix <- List("3h", "6h", "12h", "1d", "3d", "7d")) {
+                  val view = if (!d1.containsKey("ad_view_" + prefix)) 0D else d1.getIntValue("ad_view_" + prefix).toDouble
+                  val click = if (!d1.containsKey("ad_click_" + prefix)) 0D else d1.getIntValue("ad_click_" + prefix).toDouble
+                  val conver = if (!d1.containsKey("ad_conversion_" + prefix)) 0D else d1.getIntValue("ad_conversion_" + prefix).toDouble
+                  val income = if (!d1.containsKey("ad_income_" + prefix)) 0D else d1.getIntValue("ad_income_" + prefix).toDouble
+                  val cpc = if (click == 0) 0D else income / click
+                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                  val f4 = conver
+                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                  featureMap.put("d1_feature" + "_" + prefix + "_" + "ctr", f1)
+                  featureMap.put("d1_feature" + "_" + prefix + "_" + "ctcvr", f2)
+                  featureMap.put("d1_feature" + "_" + prefix + "_" + "cvr", f3)
+                  featureMap.put("d1_feature" + "_" + prefix + "_" + "conver", f4)
+                  featureMap.put("d1_feature" + "_" + prefix + "_" + "ecpm", f5)
+                }
+              }
+
+              val vidRankMaps = scala.collection.mutable.Map[String, scala.collection.immutable.Map[String, Double]]()
+              if (d2.nonEmpty) {
+                d2.foreach(r => {
+                  val key = r._1
+                  val value = d2.getString(key).split(",").map(r => {
+                    val rList = r.split(":")
+                    (rList(0), rList(2).toDouble)
+                  }).toMap
+                  vidRankMaps.put(key, value)
+                })
+              }
+              for (prefix1 <- List("ctr", "ctcvr", "ecpm")) {
+                for (prefix2 <- List("1d", "3d", "7d", "14d")) {
+                  if (vidRankMaps.contains(prefix1 + "_" + prefix2)) {
+                    val rank = vidRankMaps(prefix1 + "_" + prefix2).getOrDefault(cid, 0.0)
+                    if (rank >= 1.0) {
+                      featureMap.put("vid_rank_" + prefix1 + "_" + prefix2, 1.0 / rank)
+                    }
+                  }
+                }
+              }
+
+              if (d3.nonEmpty) {
+                val vTitle = d3.getString("title")
+                featureMap.put("cate1", d3.getOrDefault("merge_first_level_cate", ""))
+                featureMap.put("cate2", d3.getOrDefault("merge_second_level_cate", ""))
+                featureMap.put("title_split", d3.getOrDefault("title_split", ""))
+              }
+
+              // 随机mask部分特征供模型训练
+              if (Random.nextDouble() < maskFeatureRate) {
+                featureMap.put("cid", "")
+                featureMap.put("adid", "")
+                featureMap.put("adverid", "")
+                featureMap.put("customer", "")
+              }
+              featureMap.put("flag", flag)
+
+              val jList: List[(String, JSONObject, List[String])] = List(
+                ("j1", j1, List("3h", "3d")),
+                ("j2", j2, List("3h", "3d")),
+                ("j3", j3, List("3h", "3d")),
+                ("j4", j4, List("3h", "3d")),
+                ("j5", j5, List("3h", "3d")),
+                ("j6", j6, List("1h", "2h", "3h", "6h", "12h", "1d", "3d", "today", "yesterday")),
+                ("j7", j7, List("3h", "3d")),
+                ("j8", j8, List("3h", "3d")),
+                ("j9", j9, List("3h", "3d")),
+                ("j10", j10, List("3h", "3d")),
+                ("j11", j11, List("3h", "3d"))
+              )
+              for ((prefix1, bn, periods) <- jList) {
+                for (prefix2 <- periods) {
+                  val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                  val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                  val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                  val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                  val cpc = if (click == 0) 0D else income / click
+                  val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                  val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                  val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                  val f4 = conver
+                  val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
+                  featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
+                }
+              }
+
+              // k1~k4 事件特征：view/click 来自 ad_*，conver 按 targeting_conversion 取对应事件计数
+              val kList: List[(String, JSONObject)] = List(
+                ("k1", k1), ("k2", k2), ("k3", k3), ("k4", k4)
+              )
+              val kPeriods = List("2h", "4h", "6h", "12h", "1d", "3d", "today", "1w")
+              for ((kPrefix, kn) <- kList) {
+                for (period <- kPeriods) {
+                  val view = if (kn.isEmpty) 0D else kn.getIntValue("ad_view_" + period).toDouble
+                  val click = if (kn.isEmpty) 0D else kn.getIntValue("ad_click_" + period).toDouble
+                  val eventJson = parseEventJson(kn, "event_" + period)
+                  val conver =
+                    if (eventJson.isEmpty || targetingConversion.isEmpty) 0D
+                    else eventJson.getIntValue(targetingConversion + "_" + period).toDouble
+                  val ctr = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                  val cvr = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                  val ctvr = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                  val featPrefix = kPrefix + "_" + period
+                  featureMap.put(featPrefix + "_view", view)
+                  featureMap.put(featPrefix + "_click", click)
+                  featureMap.put(featPrefix + "_conver", conver)
+                  featureMap.put(featPrefix + "_ctr", ctr)
+                  featureMap.put(featPrefix + "_cvr", cvr)
+                  featureMap.put(featPrefix + "_ctvr", ctvr)
+                }
+              }
+
+              /*
+            广告
+              sparse：cid adid adverid targeting_conversion
+
+              cpa --> 1个
+              adverid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr conver ecpm  --> 30个
+              cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              地理//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              app//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              手机品牌//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              系统 无数据
+              week//cid下的 7d 14d、 ctr ctcvr cvr ecpm conver --> 10个
+              hour//cid下的 7d 14d、 ctr ctcvr cvr ecpm conver --> 10个
+
+            用户
+              用户历史 点击/转化 的title tag；3d 7d 14d； cid的title； 数量/最高分/平均分 --> 18个
+              用户历史 14d 看过/点过/转化次数/income； ctr cvr ctcvr ecpm；  --> 8个
+
+              用户到cid的ui特征 --> 10个
+                1/用户最近看过这个cid的时间间隔
+                1/用户最近点过这个cid的时间间隔
+                1/用户最近转过这个cid的时间间隔
+                用户看过这个cid多少次
+                用户点过这个cid多少次
+                用户转过这个cid多少次
+                用户对这个cid花了多少钱
+                用户对这个cid的ctr ctcvr cvr
+
+            视频
+              vid//cid下的 3h 6h 12h 1d 3d 7d 、 ctr ctcvr cvr ecpm conver --> 30个
+              vid//cid下的 1d 3d 7d 14d、 ctr ctcvr ecpm 的rank值 倒数 --> 12个
+
+             */
+
+
+              //4 处理label信息（从 label_json 读取）。
+              val labels = new JSONObject
+              val labelJson = getJsonObject(record, "label_json")
+              if (!labelJson.isEmpty) {
+                labels.put("ad_is_click", labelJson.getIntValue("ad_is_click"))
+                labels.put("ad_is_conversion", labelJson.getIntValue("ad_is_conversion"))
+              } else {
+                labels.put("ad_is_click", 0)
+                labels.put("ad_is_conversion", 0)
+              }
+              //5 处理log key表头。
+              val headvideoid = record.getString("headvideoid")
+              val logKey = (apptype, mid, cid, ts, headvideoid).productIterator.mkString(",")
+              val labelKey = labels.toString()
+              (logKey, labelKey, featureMap)
+            })
+          odpsData
+        }).reduce(_ union _)
+        .map { case (logKey, labelKey, jsons) =>
+          val denseFeatures = scala.collection.mutable.Map[String, Double]()
+          val sparseFeatures = scala.collection.mutable.Map[String, String]()
+          denseFeatureNames.foreach(r => {
+            if (jsons.containsKey(r)) {
+              denseFeatures.put(r, jsons.getDoubleValue(r))
+            }
+          })
+          sparseFeatureNames.foreach(r => {
+            if (jsons.get(r) != null) {
+              sparseFeatures.put(r, jsons.get(r).toString)
+            }
+          })
+          (logKey, labelKey, denseFeatures, sparseFeatures)
+        }
+        .map {
+          case (logKey, labelKey, denseFeatures, sparseFeatures) =>
+            val labelObject = JSON.parseObject(labelKey)
+            val label = labelObject.getOrDefault(whatLabel, "0").toString
+            val bucketsMap = bucketsMap_br.value
+            // k1~k4 新特征不受 filterNames 影响（避免 _4h_ 等把 k*_4h_* 误杀）；原有 b* 等仍按 filterNames 过滤
+            def keepDense(name: String): Boolean =
+              name.startsWith("k1_") || name.startsWith("k2_") || name.startsWith("k3_") || name.startsWith("k4_") ||
+                !filterNames.exists(name.contains)
+            var resultMap = denseFeatures.collect {
+              case (name, score) if keepDense(name) && score > 1E-8 =>
+                val value = if (bucketsMap.contains(name)) {
+                  val (bucketsNum, buckets) = bucketsMap(name)
+                  1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
+                } else {
+                  score
+                }
+                name -> value.toString
+            }.toMap
+            sparseFeatures.foreach(kv => {
+              resultMap += (kv._1 -> kv._2)
+            })
+            resultMap += ("has_click" -> labelObject.getString("ad_is_click"))
+            resultMap += ("has_conversion" -> labelObject.getString("ad_is_conversion"))
+            resultMap += ("logkey" -> logKey)
+            resultMap
+        }.coalesce(128)
+
+      val partition = s"dt=$dt"
+      if (outputTable2.isEmpty) {
+        odpsOps.saveToTable(project, outputTable, partition, recordRdd, write, defaultCreate = true, overwrite = true)
+      } else {
+        // 固定seed以保证可重入
+        val splitRdds = recordRdd.randomSplit(Array(splitRate, 1 - splitRate), seed = dt.toLong)
+        odpsOps.saveToTable(project, outputTable, partition, splitRdds(0), write, defaultCreate = true, overwrite = true)
+        odpsOps.saveToTable(project, outputTable2, partition, splitRdds(1), write, defaultCreate = true, overwrite = true)
+      }
+    }
+  }
+
+  def write(map: Map[String, String], record: Record, schema: TableSchema): Unit = {
+    for ((columnName, value) <- map) {
+      try {
+        // 查找列名在表结构中的索引
+        val columnIndex = schema.getColumnIndex(columnName.toLowerCase)
+        // 获取列的类型
+        val columnType = schema.getColumn(columnIndex).getTypeInfo
+        try {
+          columnType.getTypeName match {
+            case "STRING" =>
+              record.setString(columnIndex, value)
+            case "BIGINT" =>
+              record.setBigint(columnIndex, value.toLong)
+            case "DOUBLE" =>
+              record.setDouble(columnIndex, value.toDouble)
+            case "BOOLEAN" =>
+              record.setBoolean(columnIndex, value.toBoolean)
+            case other =>
+              throw new IllegalArgumentException(s"Unsupported column type: $other")
+          }
+        } catch {
+          case e: NumberFormatException =>
+            println(s"Error converting value $value to type ${columnType.getTypeName} for column $columnName: ${e.getMessage}")
+          case e: Exception =>
+            println(s"Unexpected error writing value $value to column $columnName: ${e.getMessage}")
+        }
+      } catch {
+        case e: IllegalArgumentException => {
+          println(e.getMessage)
+        }
+      }
+    }
+  }
+
+
+  def func(record: Record, schema: TableSchema): Record = {
+    record
+  }
+
+  def funcC34567ForTagsNew(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
+    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
+    val tagsList = tags.split(",")
+    var d1 = 0.0
+    val d2 = new ArrayBuffer[String]()
+    var d3 = 0.0
+    var d4 = 0.0
+    for (tag <- tagsList) {
+      if (title.contains(tag)) {
+        d1 = d1 + 1.0
+        d2.add(tag)
+      }
+      val score = SimilarityUtils.word2VecSimilarity(tag, title)
+      d3 = if (score > d3) score else d3
+      d4 = d4 + score
+    }
+    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
+    (d1, d2.mkString(","), d3, d4)
+  }
+
+  def getJsonObject(record: Record, name: String): JSONObject = {
+    if (record.isNull(name)) {
+      new JSONObject()
+    } else {
+      JSON.parseObject(record.getString(name))
+    }
+  }
+
+  def parseEventJson(feature: JSONObject, eventKey: String): JSONObject = {
+    if (feature == null || feature.isEmpty || !feature.containsKey(eventKey)) {
+      return new JSONObject()
+    }
+    val raw = feature.getString(eventKey)
+    if (raw == null || raw.isEmpty) {
+      return new JSONObject()
+    }
+    try {
+      val parsed = JSON.parseObject(raw)
+      if (parsed == null) new JSONObject() else parsed
+    } catch {
+      case _: Exception => new JSONObject()
+    }
+  }
+}

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20260810.scala

@@ -0,0 +1,894 @@
+package com.aliyun.odps.spark.examples.makedata_ad.v20240718
+
+import com.alibaba.fastjson.{JSON, JSONObject}
+import com.aliyun.odps.TableSchema
+import com.aliyun.odps.data.Record
+import com.aliyun.odps.spark.examples.myUtils.{ParamUtils, env}
+import examples.extractor.{ExtractorUtils, RankExtractorFeature_20240530}
+import examples.utils.{DateTimeUtil, SimilarityUtils}
+import org.apache.spark.sql.SparkSession
+
+import java.text.SimpleDateFormat
+import java.time.{Instant, ZoneId, ZonedDateTime}
+import java.util.Calendar
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
+import scala.io.Source
+import scala.language.postfixOps
+import scala.util.Random
+
+/**
+ * 20260810: 从 ad_engine_statistics_log_per5min_new 生产特征。
+ * - id 类：从表字段取（abcode/adid/adverid/apptype/cid/mid/pqtid/vid/extractstrategy）
+ * - 其余特征：从 metafeature 解析（含已并入的 j1~j11 / k1~k4）
+ * - customer_id / landing_page_type：metafeature.reqFeature.customerId / landingPageType
+ * - 不产出 label（has_click / has_conversion）
+ * - beginStr/endStr：输入分区时间 yyyyMMddHHmmss（如 20260810135000），按该区间截取 5 分钟分区
+ * - 输出仍按天分区 dt=yyyyMMdd
+ */
+object makedata_ad_33_bucketDataFromOriginToHive_20260810 {
+  val CTR_SMOOTH_BETA_FACTOR = 25
+  val CVR_SMOOTH_BETA_FACTOR = 10
+  val CTCVR_SMOOTH_BETA_FACTOR = 100
+
+  def main(args: Array[String]): Unit = {
+    val spark = SparkSession
+      .builder()
+      .appName(this.getClass.getName)
+      .getOrCreate()
+    val sc = spark.sparkContext
+
+
+    // 1 读取参数
+    val param = ParamUtils.parseArgs(args)
+    val tablePart = param.getOrElse("tablePart", "64").toInt
+    // 分区时间：yyyyMMddHHmmss，也兼容 yyyyMMdd / yyyyMMddHH
+    val beginStr = param.getOrElse("beginStr", "20260810080000")
+    val endStr = param.getOrElse("endStr", "20260810235500")
+    val project = param.getOrElse("project", "loghubods")
+    val inputTable = param.getOrElse("inputTable", "ad_engine_statistics_log_per5min_new")
+    val outputTable = param.getOrElse("outputTable", "ad_easyrec_train_data_v1_sampled")
+    val outputTable2 = param.getOrElse("outputTable2", "")
+    val filterHours = param.getOrElse("filterHours", "00,01,02,03,04,05,06,07").split(",").toSet
+    val idDefaultValue = param.getOrElse("idDefaultValue", "1.0").toDouble
+    val filterNames = param.getOrElse("filterNames", "").split(",").filter(_.nonEmpty).toSet
+    val filterAdverIds = param.getOrElse("filterAdverIds", "").split(",").filter(_.nonEmpty).toSet
+    // 分割样本集的比例，splitRate部分输出至outputTable，补集输出至outputTable2（如果outputTable2不为空）
+    val splitRate = param.getOrElse("splitRate", "0.9").toDouble
+    val maskFeatureRate = param.getOrElse("maskFeatureRate", "0.0").toDouble
+    val bucketFile = param.getOrElse("bucketFile", "20260807_ad_bucket_1112.txt")
+    val flag = param.getOrElse("flag", "0")
+
+    val loader = getClass.getClassLoader
+    val resourceUrlBucket = loader.getResource(bucketFile)
+    val buckets =
+      if (resourceUrlBucket != null) {
+        val buckets = Source.fromURL(resourceUrlBucket).getLines().mkString("\n")
+        Source.fromURL(resourceUrlBucket).close()
+        buckets
+      } else {
+        ""
+      }
+    val bucketsMap = buckets.split("\n")
+      .map(r => r.replace(" ", "").replaceAll("\n", ""))
+      .filter(r => r.nonEmpty)
+      .map(r => {
+        val rList = r.split("\t")
+        val featureName = rList(0).replace("*", "_x_").replace("(view)", "_view")
+        (featureName, (rList(1).toDouble, rList(2).split(",").map(_.toDouble)))
+      }).toMap
+    println(bucketsMap.keySet)
+    val bucketsMap_br = sc.broadcast(bucketsMap)
+    val denseFeatureNames = bucketsMap.keySet
+    val lowerCaseDenseFeatureNames = bucketsMap.keySet.map(_.toLowerCase)
+    val sparseFeatureNames = Set(
+      "cid", "adid", "adverid", "targeting_conversion",
+      "region", "city", "brand",
+      "vid", "cate1", "cate2",
+      "user_cid_click_list", "user_cid_conver_list",
+      "user_vid_return_tags_2h", "user_vid_return_tags_1d", "user_vid_return_tags_3d", "user_vid_return_tags_7d",
+      "user_vid_return_tags_14d", "apptype", "ts", "mid", "pqtid", "hour", "hour_quarter", "root_source_scene",
+      "root_source_channel", "is_first_layer", "title_split", "profession", "user_vid_share_tags_1d", "user_vid_share_tags_14d",
+      "user_vid_return_cate1_14d", "user_vid_return_cate2_14d", "user_vid_share_cate1_14d", "user_vid_share_cate2_14d",
+      "creative_type", "creative_hook_embedding", "creative_why_embedding", "creative_action_embedding", "user_has_conver_1y",
+      "user_adverid_view_3d", "user_adverid_view_7d", "user_adverid_view_30d",
+      "user_adverid_click_3d", "user_adverid_click_7d", "user_adverid_click_30d",
+      "user_adverid_conver_3d", "user_adverid_conver_7d", "user_adverid_conver_30d",
+      "user_skuid_view_3d", "user_skuid_view_7d", "user_skuid_view_30d",
+      "user_skuid_click_3d", "user_skuid_click_7d", "user_skuid_click_30d",
+      "user_skuid_conver_3d", "user_skuid_conver_7d", "user_skuid_conver_30d",
+      "is_weekday", "day_of_the_week", "user_conver_ad_class", "category_name",
+      "material_md5", "user_layer", "user_layer_l6", "user_class", "user_click_ad_class", "user_view_ad_class",
+      "customer", "customer_id", "landing", "landing_page_type", "agent_id", "flag")
+
+
+    // 2 读取odps+表信息
+    val odpsOps = env.getODPS(sc)
+
+    val tableSchema = odpsOps.getTableSchema(project, outputTable, isPartition = false)
+
+    // 检查所有字段，收集非法字段
+    val invalidFields = tableSchema.flatMap { case (fieldName, _) =>
+      // 跳过 has_click / has_conversion / logkey（本脚本不产出 label）
+      if (fieldName != "has_click" && fieldName != "has_conversion" && fieldName != "logkey") {
+        if (!lowerCaseDenseFeatureNames.contains(fieldName) && !sparseFeatureNames.contains(fieldName)) {
+          Some(fieldName) // 收集缺少字段
+        } else {
+          None
+        }
+      } else {
+        None
+      }
+    }.toList
+
+    // 如果存在非法字段，抛出标准异常
+    if (invalidFields.nonEmpty) {
+      throw new IllegalArgumentException(s"缺少字段: ${invalidFields.mkString(", ")}")
+    }
+
+    // 3 按 beginStr~endStr 截取 5 分钟分区，再按天写出
+    val partitionTimes = get5MinPartitionRange(beginStr, endStr).filter { p =>
+      val hh = p.substring(8, 10)
+      filterHours.isEmpty || !filterHours.contains(hh)
+    }
+    println(s"partition range: $beginStr ~ $endStr, count=${partitionTimes.size}")
+    val partitionsByDay = partitionTimes.groupBy(_.substring(0, 8)).toSeq.sortBy(_._1)
+    for ((dt, dayParts) <- partitionsByDay) {
+      val inputPartitions = dayParts.map(p => s"dt=$p")
+      if (inputPartitions.isEmpty) {
+        println(s"skip dt=$dt, no input partitions after filterHours")
+      } else {
+      val recordRdd = inputPartitions.map { partition =>
+          println("read partition:" + partition)
+          val odpsData = odpsOps.readTable(project = project,
+              table = inputTable,
+              partition = partition,
+              transfer = func,
+              numPartition = tablePart)
+            .filter(record => !record.isNull("metafeature") && record.getString("metafeature").nonEmpty)
+            .filter(record => Option(record.getString("extractstrategy")).contains("engine"))
+            .filter(record => {
+              val appType = Option(record.getString("apptype")).getOrElse("")
+              !Set("12", "13").contains(appType)
+            })
+            .filter(record => {
+              val adverId = Option(record.getString("adverid")).getOrElse("")
+              !filterAdverIds.contains(adverId)
+            })
+            .map(record => {
+              // id 类从表字段取；其余从 metafeature 解析
+              val mateFeature: JSONObject = parseJsonObject(record.getString("metafeature"))
+              val featureMap = new JSONObject()
+              val reqFeature: JSONObject = getJsonFromMeta(mateFeature, "reqFeature")
+              val sceneFeature: JSONObject = getJsonFromMeta(mateFeature, "sceneFeature")
+              val b1: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_basic_info")
+              val b2: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_adver_action")
+              val b3: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_cid_action")
+              val b4: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_region_action")
+              val b5: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_app_action")
+              val b6: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_week_action")
+              val b7: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_hour_action")
+              val b8: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_brand_action")
+              val b9: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_weChatVersion_action")
+              // j/k：按线上 otherFeature 原表全称从 metafeature 取
+              val j1: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_level_action")
+              val j2: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_level_advertiser_action")
+              val j3: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_level_customer_action")
+              val j4: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_level_profession_action")
+              val j5: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_level_category_action")
+              val j6: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_level_cid_action")
+              val j7: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_landingpage_action")
+              val j8: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_landingpage_advertiser_action")
+              val j9: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_landingpage_customer_action")
+              val j10: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_landingpage_profession_action")
+              val j11: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_landingpage_category_action")
+              val k1: JSONObject = getJsonFromMeta(mateFeature, "alg_feature_level6_landtype_action")
+              val k2: JSONObject = getJsonFromMeta(mateFeature, "alg_feature_level6_landtype_cid_action")
+              val k3: JSONObject = getJsonFromMeta(mateFeature, "alg_feature_level6_landtype_profession_action")
+              val k4: JSONObject = getJsonFromMeta(mateFeature, "alg_feature_level6_landtype_profession_agent_custom_action")
+
+              // id 类：直接取表字段
+              val cid = Option(record.getString("cid")).getOrElse("")
+              val adid = Option(record.getString("adid")).getOrElse("")
+              val adverId = Option(record.getString("adverid")).getOrElse("")
+              val mid = Option(record.getString("mid")).getOrElse("")
+              val pqtid = Option(record.getString("pqtid")).getOrElse("")
+              val apptype = Option(record.getString("apptype")).getOrElse("")
+              val abcode = Option(record.getString("abcode")).getOrElse("")
+              val vid = Option(record.getString("vid")).getOrElse("")
+              // ts：sceneFeature.ts
+              val tsStr = firstNonEmpty(
+                sceneFeature.getString("ts"),
+                mateFeature.getString("ts"),
+                reqFeature.getString("ts")
+              )
+              val targetingConversion = firstNonEmpty(
+                reqFeature.getString("targeting_conversion"),
+                b1.getString("targeting_conversion")
+              )
+
+              if (tsStr.isEmpty || cid.isEmpty) {
+                None
+              } else {
+                val ts = tsStr.toInt
+                val instant = Instant.ofEpochSecond(ts)
+                val chinaZone = ZoneId.of("Asia/Shanghai")
+                val zonedDateTime = ZonedDateTime.ofInstant(instant, chinaZone)
+                val dayOfTheWeek = zonedDateTime.getDayOfWeek.getValue()
+                val isWeekday = if (dayOfTheWeek <= 5) 1 else 2
+
+                featureMap.put("apptype", apptype)
+                featureMap.put("ts", ts)
+                featureMap.put("mid", mid)
+                featureMap.put("pqtid", pqtid)
+                featureMap.put("targeting_conversion", targetingConversion)
+                if (cid.nonEmpty) featureMap.put("cid", cid)
+                if (adid.nonEmpty) featureMap.put("adid", adid)
+                if (adverId.nonEmpty) featureMap.put("adverid", adverId)
+                if (vid.nonEmpty) featureMap.put("vid", vid)
+
+                featureMap.put("cid_" + cid, idDefaultValue)
+                if (adid.nonEmpty) {
+                  featureMap.put("adid_" + adid, idDefaultValue)
+                }
+                if (adverId.nonEmpty) {
+                  featureMap.put("adverid_" + adverId, idDefaultValue)
+                }
+                if (targetingConversion.nonEmpty) {
+                  featureMap.put("targeting_conversion_" + targetingConversion, idDefaultValue)
+                }
+                if (b1.containsKey("creative_type") && b1.getString("creative_type").nonEmpty) {
+                  featureMap.put("creative_type", b1.getString("creative_type"))
+                }
+                if (b1.containsKey("creative_hook_embedding") && b1.getString("creative_hook_embedding").nonEmpty) {
+                  featureMap.put("creative_hook_embedding", b1.getString("creative_hook_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
+                }
+                if (b1.containsKey("creative_why_embedding") && b1.getString("creative_why_embedding").nonEmpty) {
+                  featureMap.put("creative_why_embedding", b1.getString("creative_why_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
+                }
+                if (b1.containsKey("creative_action_embedding") && b1.getString("creative_action_embedding").nonEmpty) {
+                  featureMap.put("creative_action_embedding", b1.getString("creative_action_embedding").split('|').map(_.toDouble).map(_.toFloat).mkString("|"))
+                }
+
+                // customer / landing：reqFeature.customerId / landingPageType
+                val customerId = Option(reqFeature.getString("customerId")).getOrElse("")
+                if (customerId.nonEmpty) {
+                  featureMap.put("customer", customerId)
+                  featureMap.put("customer_id", customerId)
+                }
+                if (sceneFeature.containsKey("hour") && sceneFeature.getString("hour").nonEmpty) {
+                  featureMap.put("hour", sceneFeature.getString("hour"))
+                }
+                if (sceneFeature.containsKey("hour_quarter") && sceneFeature.getString("hour_quarter").nonEmpty) {
+                  featureMap.put("hour_quarter", sceneFeature.getString("hour_quarter"))
+                }
+                featureMap.put("is_weekday", isWeekday)
+                featureMap.put("day_of_the_week", dayOfTheWeek)
+
+                val hour = DateTimeUtil.getHourByTimestamp(ts)
+                featureMap.put("hour_" + hour, idDefaultValue)
+
+                val dayOfWeek = DateTimeUtil.getDayOrWeekByTimestamp(ts)
+                featureMap.put("dayofweek_" + dayOfWeek, idDefaultValue)
+
+                if (apptype.nonEmpty) {
+                  featureMap.put("apptype_" + apptype, idDefaultValue)
+                }
+
+                if (abcode.nonEmpty) {
+                  featureMap.put("abcode_" + abcode, idDefaultValue)
+                }
+
+                // 场景/画像等仍从 reqFeature 取；id 类已用表字段覆盖
+                val reqFeatureKeys = List(
+                  "profession", "region",
+                  "city", "is_first_layer", "root_source_scene",
+                  "root_source_channel", "brand", "category_name", "material_md5"
+                )
+                reqFeatureKeys.foreach { key =>
+                  val value = reqFeature.getString(key)
+                  if (value != null && value.nonEmpty) {
+                    featureMap.put(key, value)
+                  }
+                }
+                val landingPageType = Option(reqFeature.getString("landingPageType")).getOrElse("")
+                if (landingPageType.nonEmpty) {
+                  featureMap.put("landing", landingPageType)
+                  featureMap.put("landing_page_type", landingPageType)
+                }
+                if (reqFeature.containsKey("agentId") && reqFeature.getString("agentId").nonEmpty) {
+                  featureMap.put("agent_id", reqFeature.getString("agentId"))
+                }
+                if (reqFeature.containsKey("layer_l4")) {
+                  featureMap.put("user_layer", reqFeature.getString("layer_l4"))
+                }
+                if (reqFeature.containsKey("layer_l6") && reqFeature.getString("layer_l6").nonEmpty) {
+                  featureMap.put("user_layer_l6", reqFeature.getString("layer_l6"))
+                } else {
+                  featureMap.put("user_layer_l6", "无曝光")
+                }
+                if (reqFeature.containsKey("clazz_l4")) {
+                  featureMap.put("user_class", reqFeature.getString("clazz_l4"))
+                }
+                if (b1.containsKey("cpa")) {
+                  featureMap.put("cpa", b1.getString("cpa").toDouble)
+                }
+                if (b1.containsKey("weight") && b1.getString("weight").nonEmpty) {
+                  featureMap.put("weight", b1.getString("weight").toDouble)
+                }
+
+                for ((bn, prefix1) <- List(
+                  (b2, "b2"), (b3, "b3"), (b4, "b4"), (b5, "b5"), (b8, "b8"), (b9, "b9")
+                )) {
+                  for (prefix2 <- List(
+                    "3h", "6h", "12h", "1d", "3d", "7d", "today", "yesterday"
+                  )) {
+                    val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                    val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                    val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                    val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                    val cpc = if (click == 0) 0D else income / click
+                    val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                    val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                    val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                    val f4 = conver
+                    val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
+                  }
+                }
+
+                for ((bn, prefix1) <- List(
+                  (b6, "b6"), (b7, "b7")
+                )) {
+                  for (prefix2 <- List(
+                    "7d", "14d"
+                  )) {
+                    val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                    val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                    val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                    val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                    val cpc = if (click == 0) 0D else income / click
+                    val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                    val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                    val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                    val f4 = conver
+                    val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
+                  }
+                }
+
+
+                val c1: JSONObject = getJsonFromMeta(mateFeature, "alg_mid_feature_ad_action")
+
+                val midActionList = if (c1.containsKey("action") && c1.getString("action").nonEmpty) {
+                  c1.getString("action").split(",").map(r => {
+                    val rList = r.split(":")
+                    (rList(0), (rList(1).toInt, rList(2).toInt, rList(3).toInt, rList(4).toInt, rList(5)))
+                  }).sortBy(-_._2._1).toList
+                } else {
+                  new ArrayBuffer[(String, (Int, Int, Int, Int, String))]().toList
+                }
+                val viewAll = midActionList.size.toDouble
+                val clickAll = midActionList.map(_._2._2).sum.toDouble
+                val converAll = midActionList.map(_._2._3).sum.toDouble
+                val incomeAll = midActionList.map(_._2._4).sum.toDouble
+                featureMap.put("viewAll", viewAll)
+                featureMap.put("clickAll", clickAll)
+                featureMap.put("converAll", converAll)
+                featureMap.put("incomeAll", incomeAll)
+                featureMap.put("ctr_all", RankExtractorFeature_20240530.calDiv(clickAll, viewAll))
+                featureMap.put("ctcvr_all", RankExtractorFeature_20240530.calDiv(converAll, viewAll))
+                featureMap.put("cvr_all", RankExtractorFeature_20240530.calDiv(clickAll, converAll))
+                featureMap.put("ecpm_all", RankExtractorFeature_20240530.calDiv(incomeAll * 1000, viewAll))
+
+                if (c1.containsKey("user_has_conver_1y") && c1.getInteger("user_has_conver_1y") != null) {
+                  featureMap.put("user_has_conver_1y", c1.getInteger("user_has_conver_1y"))
+                }
+                if (c1.containsKey("user_conver_ad_class") && c1.getString("user_conver_ad_class") != null) {
+                  featureMap.put("user_conver_ad_class", c1.getString("user_conver_ad_class"))
+                }
+                if (c1.containsKey("user_click_ad_class") && c1.getString("user_click_ad_class") != null) {
+                  featureMap.put("user_click_ad_class", c1.getString("user_click_ad_class"))
+                }
+                if (c1.containsKey("user_view_ad_class") && c1.getString("user_view_ad_class") != null) {
+                  featureMap.put("user_view_ad_class", c1.getString("user_view_ad_class"))
+                }
+
+                val midTimeDiff = scala.collection.mutable.Map[String, Double]()
+                midActionList.foreach {
+                  case (cidHis, (ts_history, click, conver, income, title)) =>
+                    if (!midTimeDiff.contains("timediff_view_" + cidHis)) {
+                      midTimeDiff.put("timediff_view_" + cidHis, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                    }
+                    if (!midTimeDiff.contains("timediff_click_" + cidHis) && click > 0) {
+                      midTimeDiff.put("timediff_click_" + cidHis, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                    }
+                    if (!midTimeDiff.contains("timediff_conver_" + cidHis) && conver > 0) {
+                      midTimeDiff.put("timediff_conver_" + cidHis, 1.0 / ((ts - ts_history).toDouble / 3600.0 / 24.0))
+                    }
+                }
+
+                val midActionStatic = scala.collection.mutable.Map[String, Double]()
+                midActionList.foreach {
+                  case (cidHis, (ts_history, click, conver, income, title)) =>
+                    midActionStatic.put("actionstatic_view_" + cidHis, 1.0 + midActionStatic.getOrDefault("actionstatic_view_" + cidHis, 0.0))
+                    midActionStatic.put("actionstatic_click_" + cidHis, click + midActionStatic.getOrDefault("actionstatic_click_" + cidHis, 0.0))
+                    midActionStatic.put("actionstatic_conver_" + cidHis, conver + midActionStatic.getOrDefault("actionstatic_conver_" + cidHis, 0.0))
+                    midActionStatic.put("actionstatic_income_" + cidHis, income + midActionStatic.getOrDefault("actionstatic_income_" + cidHis, 0.0))
+                }
+
+                val clickCidList = collection.mutable.ListBuffer[String]()
+                val converCidList = collection.mutable.ListBuffer[String]()
+                midActionList.foreach {
+                  case (cidHis, (ts_history, click, conver, income, title)) =>
+                    if (click == 1) clickCidList += cidHis
+                    if (conver == 1) converCidList += cidHis
+                }
+                if (clickCidList.nonEmpty) {
+                  featureMap.put("user_cid_click_list", clickCidList.takeRight(50).mkString(","))
+                } else {
+                  featureMap.put("user_cid_click_list", "")
+                }
+                if (converCidList.nonEmpty) {
+                  featureMap.put("user_cid_conver_list", converCidList.takeRight(50).mkString(","))
+                } else {
+                  featureMap.put("user_cid_conver_list", "")
+                }
+                if (midTimeDiff.contains("timediff_view_" + cid)) {
+                  featureMap.put("timediff_view", midTimeDiff.getOrDefault("timediff_view_" + cid, 0.0))
+                }
+                if (midTimeDiff.contains("timediff_click_" + cid)) {
+                  featureMap.put("timediff_click", midTimeDiff.getOrDefault("timediff_click_" + cid, 0.0))
+                }
+                if (midTimeDiff.contains("timediff_conver_" + cid)) {
+                  featureMap.put("timediff_conver", midTimeDiff.getOrDefault("timediff_conver_" + cid, 0.0))
+                }
+                if (midActionStatic.contains("actionstatic_view_" + cid)) {
+                  featureMap.put("actionstatic_view", midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0))
+                }
+                if (midActionStatic.contains("actionstatic_click_" + cid)) {
+                  featureMap.put("actionstatic_click", midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0))
+                }
+                if (midActionStatic.contains("actionstatic_conver_" + cid)) {
+                  featureMap.put("actionstatic_conver", midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0))
+                }
+                if (midActionStatic.contains("actionstatic_income_" + cid)) {
+                  featureMap.put("actionstatic_income", midActionStatic.getOrDefault("actionstatic_income_" + cid, 0.0))
+                }
+                if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
+                  featureMap.put("actionstatic_ctr", RankExtractorFeature_20240530.calDiv(
+                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+                  ))
+                }
+                if (midActionStatic.contains("actionstatic_view_" + cid) && midActionStatic.contains("actionstatic_conver_" + cid)) {
+                  featureMap.put("actionstatic_ctcvr", RankExtractorFeature_20240530.calDiv(
+                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_view_" + cid, 0.0)
+                  ))
+                }
+                if (midActionStatic.contains("actionstatic_conver_" + cid) && midActionStatic.contains("actionstatic_click_" + cid)) {
+                  featureMap.put("actionstatic_cvr", RankExtractorFeature_20240530.calDiv(
+                    midActionStatic.getOrDefault("actionstatic_conver_" + cid, 0.0),
+                    midActionStatic.getOrDefault("actionstatic_click_" + cid, 0.0)
+                  ))
+                }
+
+                val e1: JSONObject = getJsonFromMeta(mateFeature, "alg_mid_feature_return_tags")
+                val e2: JSONObject = getJsonFromMeta(mateFeature, "alg_mid_feature_share_tags")
+                val title = b1.getOrDefault("cidtitle", "").toString
+                if (title.nonEmpty) {
+                  for ((en, prefix1) <- List((e1, "e1"), (e2, "e2"))) {
+                    for (prefix2 <- List("tags_3d", "tags_7d", "tags_14d")) {
+                      if (en.nonEmpty && en.containsKey(prefix2) && en.getString(prefix2).nonEmpty) {
+                        val (f1, f2, f3, f4) = funcC34567ForTagsNew(en.getString(prefix2), title)
+                        featureMap.put(prefix1 + "_" + prefix2 + "_matchnum", f1)
+                        featureMap.put(prefix1 + "_" + prefix2 + "_maxscore", f3)
+                        featureMap.put(prefix1 + "_" + prefix2 + "_avgscore", f4)
+                      }
+                    }
+                  }
+                }
+
+                if (e1.containsKey("tags_2h") && e1.getString("tags_2h").nonEmpty) {
+                  featureMap.put("user_vid_return_tags_2h", e1.getString("tags_2h"))
+                }
+                if (e1.containsKey("tags_1d") && e1.getString("tags_1d").nonEmpty) {
+                  featureMap.put("user_vid_return_tags_1d", e1.getString("tags_1d"))
+                }
+                if (e1.containsKey("tags_3d") && e1.getString("tags_3d").nonEmpty) {
+                  featureMap.put("user_vid_return_tags_3d", e1.getString("tags_3d"))
+                }
+                if (e1.containsKey("tags_7d") && e1.getString("tags_7d").nonEmpty) {
+                  featureMap.put("user_vid_return_tags_7d", e1.getString("tags_7d"))
+                }
+                if (e1.containsKey("tags_14d") && e1.getString("tags_14d").nonEmpty) {
+                  featureMap.put("user_vid_return_tags_14d", e1.getString("tags_14d"))
+                }
+
+                if (e2.containsKey("tags_14d") && e2.getString("tags_14d").nonEmpty) {
+                  featureMap.put("user_vid_share_tags_1d", e2.getString("tags_1d"))
+                }
+                if (e2.containsKey("tags_14d") && e2.getString("tags_14d").nonEmpty) {
+                  featureMap.put("user_vid_share_tags_14d", e2.getString("tags_14d"))
+                }
+
+                val g1: JSONObject = getJsonFromMeta(mateFeature, "mid_return_video_cate")
+                val g2: JSONObject = getJsonFromMeta(mateFeature, "mid_share_video_cate")
+                if (g1.containsKey("cate1_14d") && g1.getString("cate1_14d").nonEmpty) {
+                  featureMap.put("user_vid_return_cate1_14d", g1.getString("cate1_14d"))
+                }
+                if (g1.containsKey("cate2_14d") && g1.getString("cate2_14d").nonEmpty) {
+                  featureMap.put("user_vid_return_cate2_14d", g1.getString("cate2_14d"))
+                }
+                if (g2.containsKey("cate1_14d") && g2.getString("cate1_14d").nonEmpty) {
+                  featureMap.put("user_vid_share_cate1_14d", g2.getString("cate1_14d"))
+                }
+                if (g2.containsKey("cate2_14d") && g2.getString("cate2_14d").nonEmpty) {
+                  featureMap.put("user_vid_share_cate2_14d", g2.getString("cate2_14d"))
+                }
+
+                val h1: JSONObject = getJsonFromMeta(mateFeature, "alg_mid_feature_adver_action")
+                val h2: JSONObject = getJsonFromMeta(mateFeature, "alg_mid_feature_sku_action")
+
+                val timeDimensions = Seq("3d", "7d", "30d")
+                for (dimension <- timeDimensions) {
+                  if (h1.containsKey(dimension) && h1.getString(dimension).nonEmpty) {
+                    val action = h1.getString(dimension).split(",")
+                    if (action.length >= 3) {
+                      featureMap.put(s"user_adverid_view_${dimension}", action(0))
+                      featureMap.put(s"user_adverid_click_${dimension}", action(1))
+                      featureMap.put(s"user_adverid_conver_${dimension}", action(2))
+                    }
+                  }
+                  if (h2.containsKey(dimension) && h2.getString(dimension).nonEmpty) {
+                    val action = h2.getString(dimension).split(",")
+                    if (action.length >= 3) {
+                      featureMap.put(s"user_skuid_view_${dimension}", action(0))
+                      featureMap.put(s"user_skuid_click_${dimension}", action(1))
+                      featureMap.put(s"user_skuid_conver_${dimension}", action(2))
+                    }
+                  }
+                }
+
+
+                val d1: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_vid_cf")
+                val d2: JSONObject = getJsonFromMeta(mateFeature, "alg_cid_feature_vid_cf_rank")
+                val d3: JSONObject = getJsonFromMeta(mateFeature, "alg_vid_feature_basic_info")
+
+                if (d1.nonEmpty) {
+                  for (prefix <- List("3h", "6h", "12h", "1d", "3d", "7d")) {
+                    val view = if (!d1.containsKey("ad_view_" + prefix)) 0D else d1.getIntValue("ad_view_" + prefix).toDouble
+                    val click = if (!d1.containsKey("ad_click_" + prefix)) 0D else d1.getIntValue("ad_click_" + prefix).toDouble
+                    val conver = if (!d1.containsKey("ad_conversion_" + prefix)) 0D else d1.getIntValue("ad_conversion_" + prefix).toDouble
+                    val income = if (!d1.containsKey("ad_income_" + prefix)) 0D else d1.getIntValue("ad_income_" + prefix).toDouble
+                    val cpc = if (click == 0) 0D else income / click
+                    val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                    val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                    val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                    val f4 = conver
+                    val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                    featureMap.put("d1_feature" + "_" + prefix + "_" + "ctr", f1)
+                    featureMap.put("d1_feature" + "_" + prefix + "_" + "ctcvr", f2)
+                    featureMap.put("d1_feature" + "_" + prefix + "_" + "cvr", f3)
+                    featureMap.put("d1_feature" + "_" + prefix + "_" + "conver", f4)
+                    featureMap.put("d1_feature" + "_" + prefix + "_" + "ecpm", f5)
+                  }
+                }
+
+                val vidRankMaps = scala.collection.mutable.Map[String, scala.collection.immutable.Map[String, Double]]()
+                if (d2.nonEmpty) {
+                  d2.foreach(r => {
+                    val key = r._1
+                    val value = d2.getString(key).split(",").map(r => {
+                      val rList = r.split(":")
+                      (rList(0), rList(2).toDouble)
+                    }).toMap
+                    vidRankMaps.put(key, value)
+                  })
+                }
+                for (prefix1 <- List("ctr", "ctcvr", "ecpm")) {
+                  for (prefix2 <- List("1d", "3d", "7d", "14d")) {
+                    if (vidRankMaps.contains(prefix1 + "_" + prefix2)) {
+                      val rank = vidRankMaps(prefix1 + "_" + prefix2).getOrDefault(cid, 0.0)
+                      if (rank >= 1.0) {
+                        featureMap.put("vid_rank_" + prefix1 + "_" + prefix2, 1.0 / rank)
+                      }
+                    }
+                  }
+                }
+
+                if (d3.nonEmpty) {
+                  featureMap.put("cate1", d3.getOrDefault("merge_first_level_cate", ""))
+                  featureMap.put("cate2", d3.getOrDefault("merge_second_level_cate", ""))
+                  featureMap.put("title_split", d3.getOrDefault("title_split", ""))
+                }
+
+                if (Random.nextDouble() < maskFeatureRate) {
+                  featureMap.put("cid", "")
+                  featureMap.put("adid", "")
+                  featureMap.put("adverid", "")
+                  featureMap.put("customer", "")
+                }
+                featureMap.put("flag", flag)
+
+                val jList: List[(String, JSONObject, List[String])] = List(
+                  ("j1", j1, List("3h", "3d")),
+                  ("j2", j2, List("3h", "3d")),
+                  ("j3", j3, List("3h", "3d")),
+                  ("j4", j4, List("3h", "3d")),
+                  ("j5", j5, List("3h", "3d")),
+                  ("j6", j6, List("1h", "2h", "3h", "6h", "12h", "1d", "3d", "today", "yesterday")),
+                  ("j7", j7, List("3h", "3d")),
+                  ("j8", j8, List("3h", "3d")),
+                  ("j9", j9, List("3h", "3d")),
+                  ("j10", j10, List("3h", "3d")),
+                  ("j11", j11, List("3h", "3d"))
+                )
+                for ((prefix1, bn, periods) <- jList) {
+                  for (prefix2 <- periods) {
+                    val view = if (bn.isEmpty) 0D else bn.getIntValue("ad_view_" + prefix2).toDouble
+                    val click = if (bn.isEmpty) 0D else bn.getIntValue("ad_click_" + prefix2).toDouble
+                    val conver = if (bn.isEmpty) 0D else bn.getIntValue("ad_conversion_" + prefix2).toDouble
+                    val income = if (bn.isEmpty) 0D else bn.getIntValue("ad_income_" + prefix2).toDouble
+                    val cpc = if (click == 0) 0D else income / click
+                    val f1 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                    val f2 = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                    val f3 = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                    val f4 = conver
+                    val f5 = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR) * cpc * 1000
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctr", f1)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ctcvr", f2)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "cvr", f3)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver", f4)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "ecpm", f5)
+
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "click", click)
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_log_view", conver * RankExtractorFeature_20240530.calLog(view))
+                    featureMap.put(prefix1 + "_" + prefix2 + "_" + "conver_x_ctcvr", conver * f2)
+                  }
+                }
+
+                val kList: List[(String, JSONObject)] = List(
+                  ("k1", k1), ("k2", k2), ("k3", k3), ("k4", k4)
+                )
+                val kPeriods = List("2h", "4h", "6h", "12h", "1d", "3d", "today", "1w")
+                for ((kPrefix, kn) <- kList) {
+                  for (period <- kPeriods) {
+                    val view = if (kn.isEmpty) 0D else kn.getIntValue("ad_view_" + period).toDouble
+                    val click = if (kn.isEmpty) 0D else kn.getIntValue("ad_click_" + period).toDouble
+                    val eventJson = parseEventJson(kn, "event_" + period)
+                    val conver =
+                      if (eventJson.isEmpty || targetingConversion.isEmpty) 0D
+                      else eventJson.getIntValue(targetingConversion + "_" + period).toDouble
+                    val ctr = RankExtractorFeature_20240530.divSmooth2(click, view, CTR_SMOOTH_BETA_FACTOR)
+                    val cvr = RankExtractorFeature_20240530.divSmooth2(conver, click, CVR_SMOOTH_BETA_FACTOR)
+                    val ctvr = RankExtractorFeature_20240530.divSmooth2(conver, view, CTCVR_SMOOTH_BETA_FACTOR)
+                    val featPrefix = kPrefix + "_" + period
+                    featureMap.put(featPrefix + "_view", view)
+                    featureMap.put(featPrefix + "_click", click)
+                    featureMap.put(featPrefix + "_conver", conver)
+                    featureMap.put(featPrefix + "_ctr", ctr)
+                    featureMap.put(featPrefix + "_cvr", cvr)
+                    featureMap.put(featPrefix + "_ctvr", ctvr)
+                  }
+                }
+
+                // 不产出 label；logKey 用 pqtid（pqtid 同时作为 sparse 特征保留）
+                val logKey = pqtid
+                Some((logKey, featureMap))
+              }
+            }).collect { case Some(v) => v }
+            .map { case (logKey, jsons) =>
+              val denseFeatures = scala.collection.mutable.Map[String, Double]()
+              val sparseFeatures = scala.collection.mutable.Map[String, String]()
+              denseFeatureNames.foreach(r => {
+                if (jsons.containsKey(r)) {
+                  denseFeatures.put(r, jsons.getDoubleValue(r))
+                }
+              })
+              sparseFeatureNames.foreach(r => {
+                if (jsons.get(r) != null) {
+                  sparseFeatures.put(r, jsons.get(r).toString)
+                }
+              })
+              (logKey, denseFeatures, sparseFeatures)
+            }
+            .map {
+              case (logKey, denseFeatures, sparseFeatures) =>
+                val bucketsMap = bucketsMap_br.value
+                def keepDense(name: String): Boolean =
+                  name.startsWith("k1_") || name.startsWith("k2_") || name.startsWith("k3_") || name.startsWith("k4_") ||
+                    !filterNames.exists(name.contains)
+                var resultMap = denseFeatures.collect {
+                  case (name, score) if keepDense(name) && score > 1E-8 =>
+                    val value = if (bucketsMap.contains(name)) {
+                      val (bucketsNum, buckets) = bucketsMap(name)
+                      1.0 / bucketsNum * (ExtractorUtils.findInsertPosition(buckets, score).toDouble + 1.0)
+                    } else {
+                      score
+                    }
+                    name -> value.toString
+                }.toMap
+                sparseFeatures.foreach(kv => {
+                  resultMap += (kv._1 -> kv._2)
+                })
+                resultMap += ("logkey" -> logKey)
+                resultMap
+            }
+          odpsData
+        }.reduce(_ union _)
+        .coalesce(128)
+
+      val outPartition = s"dt=$dt"
+      if (outputTable2.isEmpty) {
+        odpsOps.saveToTable(project, outputTable, outPartition, recordRdd, write, defaultCreate = true, overwrite = true)
+      } else {
+        val splitRdds = recordRdd.randomSplit(Array(splitRate, 1 - splitRate), seed = dt.toLong)
+        odpsOps.saveToTable(project, outputTable, outPartition, splitRdds(0), write, defaultCreate = true, overwrite = true)
+        odpsOps.saveToTable(project, outputTable2, outPartition, splitRdds(1), write, defaultCreate = true, overwrite = true)
+      }
+      }
+    }
+  }
+
+  def write(map: Map[String, String], record: Record, schema: TableSchema): Unit = {
+    for ((columnName, value) <- map) {
+      try {
+        val columnIndex = schema.getColumnIndex(columnName.toLowerCase)
+        val columnType = schema.getColumn(columnIndex).getTypeInfo
+        try {
+          columnType.getTypeName match {
+            case "STRING" =>
+              record.setString(columnIndex, value)
+            case "BIGINT" =>
+              record.setBigint(columnIndex, value.toLong)
+            case "DOUBLE" =>
+              record.setDouble(columnIndex, value.toDouble)
+            case "BOOLEAN" =>
+              record.setBoolean(columnIndex, value.toBoolean)
+            case other =>
+              throw new IllegalArgumentException(s"Unsupported column type: $other")
+          }
+        } catch {
+          case e: NumberFormatException =>
+            println(s"Error converting value $value to type ${columnType.getTypeName} for column $columnName: ${e.getMessage}")
+          case e: Exception =>
+            println(s"Unexpected error writing value $value to column $columnName: ${e.getMessage}")
+        }
+      } catch {
+        case e: IllegalArgumentException => {
+          println(e.getMessage)
+        }
+      }
+    }
+  }
+
+
+  def func(record: Record, schema: TableSchema): Record = {
+    record
+  }
+
+  def funcC34567ForTagsNew(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
+    val tagsList = tags.split(",")
+    var d1 = 0.0
+    val d2 = new ArrayBuffer[String]()
+    var d3 = 0.0
+    var d4 = 0.0
+    for (tag <- tagsList) {
+      if (title.contains(tag)) {
+        d1 = d1 + 1.0
+        d2.add(tag)
+      }
+      val score = SimilarityUtils.word2VecSimilarity(tag, title)
+      d3 = if (score > d3) score else d3
+      d4 = d4 + score
+    }
+    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
+    (d1, d2.mkString(","), d3, d4)
+  }
+
+  /** 规范化为 yyyyMMddHHmmss，并向下对齐到 5 分钟。兼容 yyyyMMdd / yyyyMMddHH / yyyyMMddHHmm */
+  def normalizeTo5MinPartition(raw: String, isEnd: Boolean): String = {
+    val s = Option(raw).getOrElse("").trim
+    val padded = s.length match {
+      case 8 => if (isEnd) s + "235500" else s + "000000"
+      case 10 => if (isEnd) s + "5500" else s + "0000"
+      case 12 => s + "00"
+      case 14 => s
+      case _ =>
+        throw new IllegalArgumentException(s"invalid time: $raw, expect yyyyMMddHHmmss (or yyyyMMdd / yyyyMMddHH)")
+    }
+    val min = padded.substring(10, 12).toInt
+    val alignedMin = (min / 5) * 5
+    f"${padded.substring(0, 10)}${alignedMin}%02d00"
+  }
+
+  /** [beginStr, endStr] 闭区间，步长 5 分钟，返回 yyyyMMddHHmmss 列表 */
+  def get5MinPartitionRange(beginStr: String, endStr: String): Seq[String] = {
+    val sdf = new SimpleDateFormat("yyyyMMddHHmmss")
+    val begin = sdf.parse(normalizeTo5MinPartition(beginStr, isEnd = false))
+    val end = sdf.parse(normalizeTo5MinPartition(endStr, isEnd = true))
+    if (begin.after(end)) {
+      throw new IllegalArgumentException(s"beginStr > endStr: $beginStr > $endStr")
+    }
+    val ranges = ArrayBuffer[String]()
+    val cal = Calendar.getInstance()
+    cal.setTime(begin)
+    while (!cal.getTime.after(end)) {
+      ranges += sdf.format(cal.getTime)
+      cal.add(Calendar.MINUTE, 5)
+    }
+    ranges
+  }
+
+  def parseJsonObject(raw: String): JSONObject = {
+    if (raw == null || raw.isEmpty) {
+      new JSONObject()
+    } else {
+      try {
+        val parsed = JSON.parseObject(raw)
+        if (parsed == null) new JSONObject() else parsed
+      } catch {
+        case _: Exception => new JSONObject()
+      }
+    }
+  }
+
+  def getJsonFromMeta(meta: JSONObject, name: String): JSONObject = {
+    if (meta == null || meta.isEmpty || !meta.containsKey(name)) {
+      return new JSONObject()
+    }
+    val value = meta.get(name)
+    if (value == null) {
+      return new JSONObject()
+    }
+    value match {
+      case obj: JSONObject => obj
+      case s: String => parseJsonObject(s)
+      case _ =>
+        try {
+          val parsed = meta.getJSONObject(name)
+          if (parsed == null) parseJsonObject(String.valueOf(value)) else parsed
+        } catch {
+          case _: Exception => parseJsonObject(String.valueOf(value))
+        }
+    }
+  }
+
+  def firstNonEmpty(values: String*): String = {
+    values.find(v => v != null && v.nonEmpty).getOrElse("")
+  }
+
+  def parseEventJson(feature: JSONObject, eventKey: String): JSONObject = {
+    if (feature == null || feature.isEmpty || !feature.containsKey(eventKey)) {
+      return new JSONObject()
+    }
+    val raw = feature.getString(eventKey)
+    if (raw == null || raw.isEmpty) {
+      return new JSONObject()
+    }
+    try {
+      val parsed = JSON.parseObject(raw)
+      if (parsed == null) new JSONObject() else parsed
+    } catch {
+      case _: Exception => new JSONObject()
+    }
+  }
+}