|
@@ -15,6 +15,7 @@ import lombok.extern.slf4j.Slf4j;
|
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.stereotype.Component;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
+import org.springframework.util.StringUtils;
|
|
|
|
|
|
|
|
import java.math.BigDecimal;
|
|
import java.math.BigDecimal;
|
|
|
import java.math.RoundingMode;
|
|
import java.math.RoundingMode;
|
|
@@ -53,6 +54,9 @@ public class ContentPlatformDatastatJob {
|
|
|
@XxlJob("syncContentPlatformGzhDatastatJob")
|
|
@XxlJob("syncContentPlatformGzhDatastatJob")
|
|
|
public ReturnT<String> syncContentPlatformGzhDatastatJob(String param) {
|
|
public ReturnT<String> syncContentPlatformGzhDatastatJob(String param) {
|
|
|
String dt = DateUtil.getBeforeDayDateString("yyyyMMdd");
|
|
String dt = DateUtil.getBeforeDayDateString("yyyyMMdd");
|
|
|
|
|
+ if (StringUtils.hasText(param)) {
|
|
|
|
|
+ dt = param;
|
|
|
|
|
+ }
|
|
|
long count = getGzhDatastatCount(dt);
|
|
long count = getGzhDatastatCount(dt);
|
|
|
if (count > 0) {
|
|
if (count > 0) {
|
|
|
return ReturnT.SUCCESS;
|
|
return ReturnT.SUCCESS;
|
|
@@ -120,10 +124,11 @@ public class ContentPlatformDatastatJob {
|
|
|
size += videoList.size();
|
|
size += videoList.size();
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
-
|
|
|
|
|
- BigDecimal num = BigDecimal.valueOf(score / size);
|
|
|
|
|
- BigDecimal rounded = num.setScale(2, RoundingMode.HALF_UP);
|
|
|
|
|
- item.setScore(rounded.doubleValue());
|
|
|
|
|
|
|
+ if (size > 0) {
|
|
|
|
|
+ BigDecimal num = BigDecimal.valueOf(score / size);
|
|
|
|
|
+ BigDecimal rounded = num.setScale(2, RoundingMode.HALF_UP);
|
|
|
|
|
+ item.setScore(rounded.doubleValue());
|
|
|
|
|
+ }
|
|
|
item.setCreateTimestamp(now);
|
|
item.setCreateTimestamp(now);
|
|
|
saveList.add(item);
|
|
saveList.add(item);
|
|
|
}
|
|
}
|
|
@@ -146,10 +151,10 @@ public class ContentPlatformDatastatJob {
|
|
|
return gzhDataStatMapper.countByExample(example);
|
|
return gzhDataStatMapper.countByExample(example);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- private long getQwDatastatCount(String dt) {
|
|
|
|
|
|
|
+ private List<ContentPlatformQwDataStat> getQwDatastatCount(String dt) {
|
|
|
ContentPlatformQwDataStatExample example = new ContentPlatformQwDataStatExample();
|
|
ContentPlatformQwDataStatExample example = new ContentPlatformQwDataStatExample();
|
|
|
example.createCriteria().andDateStrEqualTo(dt);
|
|
example.createCriteria().andDateStrEqualTo(dt);
|
|
|
- return qwDataStatMapper.countByExample(example);
|
|
|
|
|
|
|
+ return qwDataStatMapper.selectByExample(example);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
private long getQwDatastatTotalCount(String dt) {
|
|
private long getQwDatastatTotalCount(String dt) {
|
|
@@ -161,14 +166,15 @@ public class ContentPlatformDatastatJob {
|
|
|
@XxlJob("syncContentPlatformQwDatastatJob")
|
|
@XxlJob("syncContentPlatformQwDatastatJob")
|
|
|
public ReturnT<String> syncContentPlatformQwDatastatJob(String param) {
|
|
public ReturnT<String> syncContentPlatformQwDatastatJob(String param) {
|
|
|
String dt = DateUtil.getBeforeDayDateString("yyyyMMdd");
|
|
String dt = DateUtil.getBeforeDayDateString("yyyyMMdd");
|
|
|
- long count = getQwDatastatCount(dt);
|
|
|
|
|
- if (count > 0) {
|
|
|
|
|
- return ReturnT.SUCCESS;
|
|
|
|
|
|
|
+ if (StringUtils.hasText(param)) {
|
|
|
|
|
+ dt = param;
|
|
|
}
|
|
}
|
|
|
List<ContentPlatformQwPlan> qwPlanList = getAllQwPlan();
|
|
List<ContentPlatformQwPlan> qwPlanList = getAllQwPlan();
|
|
|
if (CollectionUtils.isEmpty(qwPlanList)) {
|
|
if (CollectionUtils.isEmpty(qwPlanList)) {
|
|
|
return ReturnT.SUCCESS;
|
|
return ReturnT.SUCCESS;
|
|
|
}
|
|
}
|
|
|
|
|
+ List<ContentPlatformQwDataStat> existList = getQwDatastatCount(dt);
|
|
|
|
|
+ List<String> existRootSourceIds = existList.stream().map(ContentPlatformQwDataStat::getRootSourceId).collect(Collectors.toList());
|
|
|
Map<String, Long> rootSourceIdMap = qwPlanList.stream()
|
|
Map<String, Long> rootSourceIdMap = qwPlanList.stream()
|
|
|
.collect(Collectors.toMap(ContentPlatformQwPlan::getRootSourceId, ContentPlatformQwPlan::getId));
|
|
.collect(Collectors.toMap(ContentPlatformQwPlan::getRootSourceId, ContentPlatformQwPlan::getId));
|
|
|
List<Long> planIds = qwPlanList.stream().map(ContentPlatformQwPlan::getId).collect(Collectors.toList());
|
|
List<Long> planIds = qwPlanList.stream().map(ContentPlatformQwPlan::getId).collect(Collectors.toList());
|
|
@@ -191,6 +197,9 @@ public class ContentPlatformDatastatJob {
|
|
|
if (!rootSourceIds.contains(rootSourceId)) {
|
|
if (!rootSourceIds.contains(rootSourceId)) {
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (existRootSourceIds.contains(rootSourceId)) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
int firstLevelCount = Integer.parseInt((String) record.get(8));
|
|
int firstLevelCount = Integer.parseInt((String) record.get(8));
|
|
|
if (firstLevelCount == 0) {
|
|
if (firstLevelCount == 0) {
|
|
|
continue;
|
|
continue;
|
|
@@ -219,6 +228,9 @@ public class ContentPlatformDatastatJob {
|
|
|
if (!rootSourceIds.contains(rootSourceId)) {
|
|
if (!rootSourceIds.contains(rootSourceId)) {
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (existRootSourceIds.contains(rootSourceId)) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
int firstLevelCount = Integer.parseInt((String) record.get(8));
|
|
int firstLevelCount = Integer.parseInt((String) record.get(8));
|
|
|
if (firstLevelCount == 0) {
|
|
if (firstLevelCount == 0) {
|
|
|
continue;
|
|
continue;
|
|
@@ -253,25 +265,32 @@ public class ContentPlatformDatastatJob {
|
|
|
@XxlJob("syncContentPlatformQwDatastatTotalJob")
|
|
@XxlJob("syncContentPlatformQwDatastatTotalJob")
|
|
|
public ReturnT<String> syncContentPlatformQwDatastatTotalJob(String param) {
|
|
public ReturnT<String> syncContentPlatformQwDatastatTotalJob(String param) {
|
|
|
String dt = DateUtil.getBeforeDayDateString("yyyyMMdd");
|
|
String dt = DateUtil.getBeforeDayDateString("yyyyMMdd");
|
|
|
- long count = getQwDatastatTotalCount(dt);
|
|
|
|
|
- if (count > 0) {
|
|
|
|
|
- return ReturnT.SUCCESS;
|
|
|
|
|
|
|
+ if (StringUtils.hasText(param)) {
|
|
|
|
|
+ dt = param;
|
|
|
}
|
|
}
|
|
|
|
|
+ dataStatMapperExt.deleteQwDatastatTotal(dt);
|
|
|
Map<String, ContentPlatformQwDataStatTotal> saveMap = new HashMap<>();
|
|
Map<String, ContentPlatformQwDataStatTotal> saveMap = new HashMap<>();
|
|
|
String outSql = String.format("SELECT * FROM loghubods.qw_out_touliu_behavior_total WHERE dt=%s;", dt);
|
|
String outSql = String.format("SELECT * FROM loghubods.qw_out_touliu_behavior_total WHERE dt=%s;", dt);
|
|
|
List<Record> outDataList = OdpsUtil.getOdpsData(outSql);
|
|
List<Record> outDataList = OdpsUtil.getOdpsData(outSql);
|
|
|
Long now = System.currentTimeMillis();
|
|
Long now = System.currentTimeMillis();
|
|
|
if (CollectionUtils.isNotEmpty(outDataList)) {
|
|
if (CollectionUtils.isNotEmpty(outDataList)) {
|
|
|
for (Record record : outDataList) {
|
|
for (Record record : outDataList) {
|
|
|
- ContentPlatformQwDataStatTotal item = new ContentPlatformQwDataStatTotal();
|
|
|
|
|
|
|
+ String type = (String) record.get(1);
|
|
|
|
|
+ if (!"社群".equals(type) && !"自动回复".equals(type)) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
String channel = (String) record.get(33);
|
|
String channel = (String) record.get(33);
|
|
|
int firstLevelCount = Integer.parseInt((String) record.get(6));
|
|
int firstLevelCount = Integer.parseInt((String) record.get(6));
|
|
|
if (firstLevelCount == 0) {
|
|
if (firstLevelCount == 0) {
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
|
|
|
+ ContentPlatformQwDataStatTotal item = saveMap.getOrDefault(channel, new ContentPlatformQwDataStatTotal());
|
|
|
item.setDateStr(dt);
|
|
item.setDateStr(dt);
|
|
|
item.setChannel(channel);
|
|
item.setChannel(channel);
|
|
|
- item.setFirstLevelCount(firstLevelCount);
|
|
|
|
|
|
|
+ if (item.getFirstLevelCount() == null) {
|
|
|
|
|
+ item.setFirstLevelCount(0);
|
|
|
|
|
+ }
|
|
|
|
|
+ item.setFirstLevelCount(item.getFirstLevelCount() + firstLevelCount);
|
|
|
item.setCreateTimestamp(now);
|
|
item.setCreateTimestamp(now);
|
|
|
saveMap.put(channel, item);
|
|
saveMap.put(channel, item);
|
|
|
}
|
|
}
|
|
@@ -279,17 +298,29 @@ public class ContentPlatformDatastatJob {
|
|
|
String out2Sql = String.format("SELECT * FROM loghubods.qw_out2_touliu_behavior_total WHERE dt=%s;", dt);
|
|
String out2Sql = String.format("SELECT * FROM loghubods.qw_out2_touliu_behavior_total WHERE dt=%s;", dt);
|
|
|
List<Record> out2DataList = OdpsUtil.getOdpsData(out2Sql);
|
|
List<Record> out2DataList = OdpsUtil.getOdpsData(out2Sql);
|
|
|
if (CollectionUtils.isNotEmpty(out2DataList)) {
|
|
if (CollectionUtils.isNotEmpty(out2DataList)) {
|
|
|
|
|
+ List<String> out2Channels = new ArrayList<>();
|
|
|
for (Record record : out2DataList) {
|
|
for (Record record : out2DataList) {
|
|
|
- ContentPlatformQwDataStatTotal item = new ContentPlatformQwDataStatTotal();
|
|
|
|
|
|
|
+ String type = (String) record.get(1);
|
|
|
|
|
+ if (!"社群".equals(type) && !"自动回复".equals(type)) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
String channel = (String) record.get(33);
|
|
String channel = (String) record.get(33);
|
|
|
int firstLevelCount = Integer.parseInt((String) record.get(6));
|
|
int firstLevelCount = Integer.parseInt((String) record.get(6));
|
|
|
if (firstLevelCount == 0) {
|
|
if (firstLevelCount == 0) {
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
|
|
|
+ if (saveMap.containsKey(channel) && !out2Channels.contains(channel)) {
|
|
|
|
|
+ saveMap.remove(channel);
|
|
|
|
|
+ }
|
|
|
|
|
+ ContentPlatformQwDataStatTotal item = saveMap.getOrDefault(channel, new ContentPlatformQwDataStatTotal());
|
|
|
item.setDateStr(dt);
|
|
item.setDateStr(dt);
|
|
|
item.setChannel(channel);
|
|
item.setChannel(channel);
|
|
|
- item.setFirstLevelCount(firstLevelCount);
|
|
|
|
|
|
|
+ if (item.getFirstLevelCount() == null) {
|
|
|
|
|
+ item.setFirstLevelCount(0);
|
|
|
|
|
+ }
|
|
|
|
|
+ item.setFirstLevelCount(item.getFirstLevelCount() + firstLevelCount);
|
|
|
item.setCreateTimestamp(now);
|
|
item.setCreateTimestamp(now);
|
|
|
|
|
+ out2Channels.add(channel);
|
|
|
saveMap.put(channel, item);
|
|
saveMap.put(channel, item);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|