|
|
@@ -173,7 +173,8 @@ object makedata_ad_33_bucketDataFromOriginToHive_20260808 {
|
|
|
!filterAdverIds.contains(adverId)
|
|
|
})
|
|
|
.filter(record => {
|
|
|
- val label = record.getString(whatLabel).toInt
|
|
|
+ val labelJson = getJsonObject(record, "label_json")
|
|
|
+ val label = labelJson.getIntValue(whatLabel)
|
|
|
label > 0 || Random.nextDouble() < negSampleRate
|
|
|
})
|
|
|
.map(record => {
|
|
|
@@ -754,12 +755,15 @@ object makedata_ad_33_bucketDataFromOriginToHive_20260808 {
|
|
|
*/
|
|
|
|
|
|
|
|
|
- //4 处理label信息。
|
|
|
+ //4 处理label信息(从 label_json 读取)。
|
|
|
val labels = new JSONObject
|
|
|
- for (labelKey <- List("ad_is_click", "ad_is_conversion")) {
|
|
|
- if (!record.isNull(labelKey)) {
|
|
|
- labels.put(labelKey, record.getString(labelKey))
|
|
|
- }
|
|
|
+ val labelJson = getJsonObject(record, "label_json")
|
|
|
+ if (!labelJson.isEmpty) {
|
|
|
+ labels.put("ad_is_click", labelJson.getIntValue("ad_is_click"))
|
|
|
+ labels.put("ad_is_conversion", labelJson.getIntValue("ad_is_conversion"))
|
|
|
+ } else {
|
|
|
+ labels.put("ad_is_click", 0)
|
|
|
+ labels.put("ad_is_conversion", 0)
|
|
|
}
|
|
|
//5 处理log key表头。
|
|
|
val headvideoid = record.getString("headvideoid")
|