|
@@ -76,18 +76,18 @@ public class ChannelDemandMatchJob {
|
|
|
private static final int MATCH_THREAD_POOL_SIZE = 5;
|
|
private static final int MATCH_THREAD_POOL_SIZE = 5;
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * 先验需求-场景 数据源策略标识
|
|
|
|
|
|
|
+ * 人群需求-场景 数据源策略标识
|
|
|
*/
|
|
*/
|
|
|
- private static final String DIMENSION_STAT_STRATEGY = "先验需求-场景";
|
|
|
|
|
|
|
+ private static final String DIMENSION_STAT_STRATEGY = "人群需求-场景";
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * 先验需求-场景 数据源的ROV阈值 (默认3%)
|
|
|
|
|
|
|
+ * 人群需求-场景 数据源的ROV阈值 (默认3%)
|
|
|
*/
|
|
*/
|
|
|
@Value("${channel.demand.dimension-stat.min-rov:0.03}")
|
|
@Value("${channel.demand.dimension-stat.min-rov:0.03}")
|
|
|
private double dimensionStatMinRov;
|
|
private double dimensionStatMinRov;
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * 先验需求-场景 数据源的UV占比阈值 (默认0.2%)
|
|
|
|
|
|
|
+ * 人群需求-场景 数据源的UV占比阈值 (默认0.2%)
|
|
|
*/
|
|
*/
|
|
|
@Value("${channel.demand.dimension-stat.min-uv-ratio:0.002}")
|
|
@Value("${channel.demand.dimension-stat.min-uv-ratio:0.002}")
|
|
|
private double dimensionStatMinUvRatio;
|
|
private double dimensionStatMinUvRatio;
|
|
@@ -179,7 +179,7 @@ public class ChannelDemandMatchJob {
|
|
|
// 1. 先清理该渠道+日期的历史数据(支持重跑)
|
|
// 1. 先清理该渠道+日期的历史数据(支持重跑)
|
|
|
deleteExistingResults(config.getId(), dt);
|
|
deleteExistingResults(config.getId(), dt);
|
|
|
|
|
|
|
|
- // 如果是先验需求-场景数据源,走独立的处理逻辑(不需要向量召回)
|
|
|
|
|
|
|
+ // 如果是人群需求-场景数据源,走独立的处理逻辑(不需要向量召回)
|
|
|
if (DIMENSION_STAT_STRATEGY.equals(config.getDemandStrategy())) {
|
|
if (DIMENSION_STAT_STRATEGY.equals(config.getDemandStrategy())) {
|
|
|
processDimensionStatSource(config, dt, totalDemands, totalMatched, totalFailed);
|
|
processDimensionStatSource(config, dt, totalDemands, totalMatched, totalFailed);
|
|
|
return;
|
|
return;
|
|
@@ -521,14 +521,14 @@ public class ChannelDemandMatchJob {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * 处理先验需求-场景数据源:从dwd_channel_element_dimension_stat查询,直接入库(不需向量召回)
|
|
|
|
|
|
|
+ * 处理人群需求-场景数据源:从dwd_channel_element_dimension_stat查询,直接入库(不需向量召回)
|
|
|
*/
|
|
*/
|
|
|
private void processDimensionStatSource(ChannelDemandMatchConfig config, String dt,
|
|
private void processDimensionStatSource(ChannelDemandMatchConfig config, String dt,
|
|
|
AtomicInteger totalDemands, AtomicInteger totalMatched, AtomicInteger totalFailed) {
|
|
AtomicInteger totalDemands, AtomicInteger totalMatched, AtomicInteger totalFailed) {
|
|
|
- log.info("开始处理先验需求-场景数据源, configId={}, dt={}", config.getId(), dt);
|
|
|
|
|
|
|
+ log.info("开始处理人群需求-场景数据源, configId={}, dt={}", config.getId(), dt);
|
|
|
|
|
|
|
|
String sql = buildDimensionStatSql(config, dt);
|
|
String sql = buildDimensionStatSql(config, dt);
|
|
|
- log.info("先验需求-场景ODPS SQL长度: {}", sql.length());
|
|
|
|
|
|
|
+ log.info("人群需求-场景ODPS SQL长度: {}", sql.length());
|
|
|
|
|
|
|
|
List<ChannelDemandMatchResult> results = new ArrayList<>();
|
|
List<ChannelDemandMatchResult> results = new ArrayList<>();
|
|
|
|
|
|
|
@@ -566,7 +566,7 @@ public class ChannelDemandMatchJob {
|
|
|
try {
|
|
try {
|
|
|
result.setMatchVideoId(Long.parseLong(videoIdStr.trim()));
|
|
result.setMatchVideoId(Long.parseLong(videoIdStr.trim()));
|
|
|
} catch (NumberFormatException e) {
|
|
} catch (NumberFormatException e) {
|
|
|
- log.warn("先验需求-场景 videoid解析失败: {}", videoIdStr);
|
|
|
|
|
|
|
+ log.warn("人群需求-场景 videoid解析失败: {}", videoIdStr);
|
|
|
return;
|
|
return;
|
|
|
}
|
|
}
|
|
|
} else {
|
|
} else {
|
|
@@ -588,11 +588,11 @@ public class ChannelDemandMatchJob {
|
|
|
results.add(result);
|
|
results.add(result);
|
|
|
}
|
|
}
|
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
|
- log.error("解析先验需求-场景ODPS记录失败: {}", e.getMessage());
|
|
|
|
|
|
|
+ log.error("解析人群需求-场景ODPS记录失败: {}", e.getMessage());
|
|
|
}
|
|
}
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- log.info("先验需求-场景数据源查询到 {} 条记录", results.size());
|
|
|
|
|
|
|
+ log.info("人群需求-场景数据源查询到 {} 条记录", results.size());
|
|
|
totalDemands.addAndGet(results.size());
|
|
totalDemands.addAndGet(results.size());
|
|
|
|
|
|
|
|
if (results.isEmpty()) {
|
|
if (results.isEmpty()) {
|
|
@@ -604,11 +604,11 @@ public class ChannelDemandMatchJob {
|
|
|
resultMapperExt.batchInsert(partition);
|
|
resultMapperExt.batchInsert(partition);
|
|
|
}
|
|
}
|
|
|
totalMatched.addAndGet(results.size());
|
|
totalMatched.addAndGet(results.size());
|
|
|
- log.info("先验需求-场景数据源写入完成, 共 {} 条", results.size());
|
|
|
|
|
|
|
+ log.info("人群需求-场景数据源写入完成, 共 {} 条", results.size());
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * 构造先验需求-场景数据源的ODPS SQL
|
|
|
|
|
|
|
+ * 构造人群需求-场景数据源的ODPS SQL
|
|
|
* 查询loghubods.dwd_channel_element_dimension_stat,过滤rov>3%且uv占比>0.2%
|
|
* 查询loghubods.dwd_channel_element_dimension_stat,过滤rov>3%且uv占比>0.2%
|
|
|
*/
|
|
*/
|
|
|
private String buildDimensionStatSql(ChannelDemandMatchConfig config, String dt) {
|
|
private String buildDimensionStatSql(ChannelDemandMatchConfig config, String dt) {
|