sort & filter

Makefile

@@ -1,4 +1,5 @@
 BUILD=go build -mod vendor
+#BUILD=go build -mod mod
 DOCKER?=docker
 SOURCE_DIR=src
 BIN_NAME=tzld_rec

conf/config.json.production

@@ -36,31 +36,31 @@
                 "Name" : "app_recommend_status",
                 "Type" : "int",
                 "Operator" : "in",
-                "Value" : [-6, 1, 10] 
+                "Value" : [-6, 1, 10]
             },
             {
                 "Name" : "audit_status",
                 "Type" : "int",
                 "Operator" : "equal",
-                "Value" : 5 
+                "Value" : 5
             },
             {
                 "Name" : "video_status",
                 "Type" : "int",
                 "Operator" : "equal",
-                "Value" : 1 
+                "Value" : 1
             },
             {
                 "Name" : "pwd_status",
                 "Type" : "int",
                 "Operator" : "equal",
-                "Value" : 0 
+                "Value" : 0
             },
             {
                 "Name" : "charge",
                 "Type" : "int",
                 "Operator" : "equal",
-                "Value" : 0 
+                "Value" : 0
             }
         ]
     },
@@ -177,7 +177,7 @@
 	"FilterNames": {
 	  "default": [
 		"UniqueFilter",
-        "UserExposureFilter", 
+        "UserExposureFilter",
         "ItemStateFilter",
         "PriorityAdjustCountFilter"
 	  ]
@@ -261,13 +261,26 @@
                     "FeatureStore":"user"
                 }
               ]
+            },
+            {
+                "FeatureDaoConf": {
+                    "AdapterType": "hologres",
+                    "HologresName": "wx-holo",
+                    "ItemFeatureKeyName" :"id",
+                    "FeatureKey": "item:id",
+                    "HologresTableName": "wx_video",
+                    "ItemSelectFields":"id, uid, total_time",
+                    "FeatureStore":"item"
+                },
+                "Features" :[
+                ]
             }
           ]
         }
 	},
     "ColdStartRankConfs" :{
         "video_feed": {
-           "RecallName": "ColdStartRecall" 
+           "RecallName": "ColdStartRecall"
         }
     }
  }

pairec/go.mod

@@ -0,0 +1,33 @@
+module gitlab.alibaba-inc.com/pai_biz_arch/pairec
+
+go 1.16
+
+require (
+	github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20191105111628-27aadc495f68
+	github.com/aliyun/aliyun-datahub-sdk-go v0.1.5
+	github.com/aliyun/aliyun-tablestore-go-sdk v4.1.3+incompatible
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/goburrow/cache v0.1.3
+	github.com/gogo/protobuf v1.3.1
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/golang/protobuf v1.4.2
+	github.com/golang/snappy v0.0.1
+	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/huandu/go-sqlbuilder v1.7.0
+	github.com/lib/pq v1.9.0
+	github.com/mailru/easyjson v0.7.6
+	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/prometheus/client_golang v1.7.1
+	github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b
+	github.com/segmentio/kafka-go v0.4.9
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
+	github.com/spaolacci/murmur3 v1.1.0
+	github.com/tsuna/gohbase v0.0.0-20200414231402-ae5d5a2cd032
+	gitlab.alibaba-inc.com/pai_biz_arch/pairec-experiment/client_golang v0.0.0-20210807034701-52a5c464d8cd
+	gonum.org/v1/gonum v0.8.2
+	google.golang.org/grpc v1.31.0
+	google.golang.org/protobuf v1.25.0
+	modernc.org/mathutil v1.1.1 // indirect
+	modernc.org/strutil v1.1.0 // indirect
+)

src/filter/duration_filter.go

@@ -0,0 +1,41 @@
+package filter
+
+import (
+	pfilter "gitlab.alibaba-inc.com/pai_biz_arch/pairec/filter"
+	"errors"
+	"gitlab.alibaba-inc.com/pai_biz_arch/pairec/module"
+)
+
+// remove duplicate item
+type DurationFilter struct {
+}
+
+func NewDurationFilter() *DurationFilter {
+	filter := DurationFilter{}
+
+	return &filter
+}
+func (f *DurationFilter) Filter(filterData *pfilter.FilterData) error {
+	if _, ok := filterData.Data.([]*module.Item); !ok {
+		return errors.New("filter data type error")
+
+	}
+	return f.doFilter(filterData)
+}
+
+func (f *DurationFilter) doFilter(filterData *pfilter.FilterData) error {
+	items := filterData.Data.([]*module.Item)
+	newItems := make([]*module.Item, 0)
+	for _, item := range items{
+		if item.Properties["total_time"].(int) > 120 {
+			newItems = append(newItems, item)
+		}
+	}
+
+	filterData.Data = newItems
+	return nil
+}
+func (f *DurationFilter) MatchTag(tag string) bool {
+	// default filter, so filter all tag
+	return true
+}

src/sort/unique_uploader_sort.go

@@ -0,0 +1,40 @@
+package sort
+
+import (
+	"errors"
+
+	"gitlab.alibaba-inc.com/pai_biz_arch/pairec/module"
+	psort "gitlab.alibaba-inc.com/pai_biz_arch/pairec/sort"
+)
+
+type SUniqueUploaderSort struct {
+}
+
+func UniqueUploaderSort() *SUniqueUploaderSort {
+	return &SUniqueUploaderSort{}
+}
+func (s *SUniqueUploaderSort) Sort(sortData *psort.SortData) error {
+	if _, ok := sortData.Data.([]*module.Item); !ok {
+		return errors.New("sort data type error")
+
+	}
+	return s.doSort(sortData)
+}
+
+// 将已存在的up主的视频过滤掉，不连续出现
+func (s *SUniqueUploaderSort) doSort(sortData *psort.SortData) error {
+	items := sortData.Data.([]*module.Item)
+	newItems := make([] *module.Item, 0)
+	uniq := make(map[string]bool)
+	for _, i := range items {
+		uploader := i.Properties["uid"].(string)
+		//if unq[uploader] {
+		if _, ok := uniq[uploader]; !ok{
+			uniq[uploader] = true
+			newItems = append(newItems, i)
+		}
+	}
+	sortData.Data = newItems
+
+	return nil
+}

vendor/gitlab.alibaba-inc.com/pai_biz_arch/pairec-experiment/client_golang/api/api_experiment.go

@@ -117,7 +117,6 @@ func (a *ExperimentApiService) CloneExperiment(ctx context.Context, experimentId
 
 	// to determine the Content-Type header
 	localVarHttpContentTypes := []string{}
-
 	// set Content-Type header
 	localVarHttpContentType := selectHeaderContentType(localVarHttpContentTypes)
 	if localVarHttpContentType != "" {