all

src/main/scala/com/aliyun/odps/spark/examples/makedata/makedata_02_writeredis.scala

@@ -38,6 +38,7 @@ object makedata_02_writeredis {
     val savePathUser = param.getOrDefault("savePathUser", "")
     val savePathVideo = param.getOrDefault("savePathVideo", "")
     val userSampleIDs = param.getOrDefault("userSampleIDs", "")
+    val sampleRate = param.getOrDefault("sampleRate", "1.0").toDouble
 //    val userSampleIDsPathFix = param.getOrDefault("userSampleIDsPathFix", "")
     //  /dw/recommend/model/feature/
 
@@ -127,7 +128,9 @@ object makedata_02_writeredis {
       if (userSampleIDs.nonEmpty) {
         savePathPart = savePathPart + "_" + userSampleIDs
       }
-      val userDataRead = sc.textFile(savePathPart).filter(_.split("\t").length >= 2).map(r => {
+      val userDataRead = sc.textFile(savePathPart).filter(_.split("\t").length >= 2)
+        .sample(false, sampleRate)
+        .map(r => {
         val rList = r.split("\t")
         (rList(0), rList(1))
       })

zhangbo/04_upload.sh

@@ -1 +1,5 @@
-dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_sharev2_20231220.txt oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/
+
+cat /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_sharev2_20231220.txt | sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' > /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_sharev2_20231220_change.txt
+
+
+dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_sharev2_20231220_change.txt oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/