Merge branch 'feature_gufengshou_20240314_pid_v4' into pre-master

ad-engine-server/src/main/java/com/tzld/piaoquan/ad/engine/server/controller/AdRecommendController.java

@@ -49,4 +49,14 @@ public class AdRecommendController {
         return map;
     }
 
+    @RequestMapping("/top1/bid/new/pid")
+    public Map<String,Object> adRecommendTop1BidNewPid(@RequestBody BidRankRecommendRequestParam request){
+        AdPlatformBidCreativeDTO rankResult = rankService.adBidRankNewPid(request);
+        HashMap map =new HashMap();
+        map.put("code","0");
+        map.put("msg","success");
+        map.put("content",rankResult);
+        return map;
+    }
+
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/RankService.java

@@ -16,5 +16,7 @@ public interface RankService {
     AdRankItem adItemRank(RankRecommendRequestParam request);
     AdPlatformBidCreativeDTO adBidRank(BidRankRecommendRequestParam request);
 
+    AdPlatformBidCreativeDTO adBidRankNewPid(BidRankRecommendRequestParam request);
+
     List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems, String configFile);
 }

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/container/PidLambdaV2Container.java

@@ -0,0 +1,171 @@
+package com.tzld.piaoquan.ad.engine.service.score.container;
+
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentialProvider;
+import com.aliyun.oss.model.CopyObjectResult;
+import com.aliyun.oss.model.OSSObject;
+import com.aliyun.oss.model.PutObjectResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+@Component
+public class PidLambdaV2Container {
+    private final static Logger log = LoggerFactory.getLogger(PidLambdaV2Container.class);
+
+    private static final int SCHEDULE_PERIOD = 10;
+    private static final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+    @Value("${model.oss.internal.endpoint:oss-cn-hangzhou.aliyuncs.com}")
+    String endpoint = "";
+    @Value("${model.oss.accessKeyId:LTAIP6x1l3DXfSxm}")
+    String accessKeyId = "";
+    @Value("${model.oss.accessKetSecret:KbTaM9ars4OX3PMS6Xm7rtxGr1FLon}")
+    String accessKetSecret = "";
+    @Value("${model.oss.bucketName:art-recommend}")
+    String bucketName = "";
+
+    @Value("${model.oss.pid.v2.filename.lambda:pid/lambdaV2.txt}")
+    String lambdaFileName = "";
+
+    @Value("${model.oss.pid.v2.filename.dCpa:pid/dCpaV2.txt}")
+    String dCpaFileName = "";
+
+    @Value("${ad.model.pid.v2.lambda.max:5.0}")
+    Double maxLambda = 0d;
+
+    @Value("${ad.model.pid.v2.lambda.min:0.8}")
+    Double minLambda = 0d;
+    OSS client;
+
+    private static ConcurrentHashMap<Long,Double>  lambdaCache=new ConcurrentHashMap<>();
+    private Date cacheDate;
+
+    @PostConstruct
+    private void init(){
+        instanceClient();
+        final Runnable task = new Runnable() {
+            public void run() {
+                try {
+                    loadAndCalIfNeed();
+                }catch (Exception e){
+                    e.printStackTrace();
+                }
+            }
+        };
+        scheduler.scheduleAtFixedRate(task, 0, SCHEDULE_PERIOD, TimeUnit.MINUTES); // 10分钟
+    }
+
+    private void instanceClient(){
+        CredentialsProvider credentialsProvider = new DefaultCredentialProvider(accessKeyId, accessKetSecret);
+        this.client = new OSSClientBuilder().build(endpoint, credentialsProvider);
+    }
+
+    private void loadAndCalIfNeed(){
+        loadLambdaFile();
+        OSSObject dCpaFileOjb=client.getObject(bucketName,dCpaFileName);
+        if(cacheDate==null||dCpaFileOjb.getObjectMetadata().getLastModified().after(cacheDate)){
+            calNewLambda(dCpaFileOjb);
+            writeLambdaFileToOss();
+        }
+    }
+
+    private void calNewLambda(OSSObject object) {
+        try {
+            InputStream is=object.getObjectContent();
+            InputStreamReader isr=new InputStreamReader(is);
+            BufferedReader bufferedReader = new BufferedReader(isr);
+            String line = null;
+            ConcurrentHashMap<Long,Double>  temp=new ConcurrentHashMap<>();
+            while ((line = bufferedReader.readLine()) != null){
+                try {
+                    String[] cols=line.split(",");
+                    Long creativeId=Long.parseLong(cols[0]);
+                    Double conversion=Double.parseDouble(cols[1]);
+                    Double cpa=Double.parseDouble(cols[2]);
+                    Double realCost=Double.parseDouble(cols[3]);
+                    Double lambdaNew=1d;
+                    if(((cpa*conversion)==0&&realCost.equals(0d))){
+                        lambdaNew=cpa*conversion/realCost;
+                    }
+
+                    if(lambdaNew>maxLambda){
+                        lambdaNew=maxLambda;
+                    }else if(lambdaNew<minLambda){
+                        lambdaNew=maxLambda;
+                    }
+                    temp.put(creativeId,lambdaNew);
+
+                    log.info("svc=calNewLambdaV2 creativeId={} lambdaNew={}", creativeId,lambdaNew);
+                }catch (Exception e){
+                    e.printStackTrace();
+                }
+            }
+            lambdaCache.clear();
+            lambdaCache=temp;
+        }catch (Exception e){
+            log.error("svc=calNewLambdaV2 status=failed error={}", Arrays.toString(e.getStackTrace()));
+        }
+    }
+
+    private void writeLambdaFileToOss(){
+        //先不考虑各种更新失败及重复更新问题。
+        try {
+            String tempFile=lambdaFileName+"_temp";
+            String content= JSONObject.toJSONString(lambdaCache);
+            PutObjectResult putObjectResult=client.putObject(bucketName,tempFile,new ByteArrayInputStream(content.getBytes()));
+            CopyObjectResult copyObjectResult=client.copyObject(bucketName, tempFile, bucketName, lambdaFileName);
+            this.cacheDate= copyObjectResult.getLastModified();
+            client.deleteObject(bucketName, tempFile);
+        }catch (Exception e){
+            log.error("svc=writeLambdaV2FileToOss status=failed error={}", Arrays.toString(e.getStackTrace()));
+            e.printStackTrace();
+        }
+    }
+
+    private void loadLambdaFile(){
+        try {
+            OSSObject object=client.getObject(bucketName,lambdaFileName);
+            if(object==null) return;
+            if(cacheDate!=null&& !cacheDate.before(object.getObjectMetadata().getLastModified())) return;
+            StringBuilder builder=new StringBuilder();
+            InputStream is=object.getObjectContent();
+            InputStreamReader isr=new InputStreamReader(is);
+            BufferedReader bufferedReader = new BufferedReader(isr);
+            String line = null;
+            while ((line=bufferedReader.readLine())!=null){
+                builder.append(line);
+            }
+            lambdaCache=JSONObject.parseObject(builder.toString(),new TypeReference<ConcurrentHashMap<Long,Double>>(){});
+            this.cacheDate=object.getObjectMetadata().getLastModified();
+        }catch (Exception e){
+            log.error("svc=loadLambdaV2File status=failed error={}", Arrays.toString(e.getStackTrace()));
+            e.printStackTrace();
+        }
+    }
+
+    public static Double getPidLambda(Long creativeId){
+        try {
+            return lambdaCache.getOrDefault(creativeId,1d);
+        }catch (Exception e){
+            return 1d;
+        }
+    }
+
+}

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceImpl.java

@@ -3,6 +3,7 @@ package com.tzld.piaoquan.ad.engine.service.score.impl;
 import com.alibaba.fastjson.JSONObject;
 import com.tzld.piaoquan.ad.engine.service.score.container.AdCreativeFeatureContainer;
 import com.tzld.piaoquan.ad.engine.service.score.container.PidLambdaContainer;
+import com.tzld.piaoquan.ad.engine.service.score.container.PidLambdaV2Container;
 import com.tzld.piaoquan.ad.engine.service.score.dto.AdPlatformBidCreativeDTO;
 import com.tzld.piaoquan.ad.engine.service.score.param.BidRankRecommendRequestParam;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdItemFeature;
@@ -200,14 +201,114 @@ public class RankServiceImpl implements RankService {
         object.put("pcvr",topItem.getCvr());
         object.put("lrsamples",topItem.getLrSampleString());
         object.put("pidLambda",topItem.getPidLambda());
+        //临时加入供pid v2使用
+        object.put("realECpm",realECpm);
+        object.put("creativeId",result.getCreativeId());
+        object.put("cpa",topItem.getCpa());
         object.put("dataTime",currentTime.format(timeFormatter));
         log.info("svc=adBidRank {}", JSONObject.toJSONString(object));
         object.remove("lrsamples");
         log.info("svc=pid_log obj={}", JSONObject.toJSONString(object));
+        log.info("svc=pid_v2_log obj={}", JSONObject.toJSONString(object));
         return result;
     }
 
+    @Override
+    public AdPlatformBidCreativeDTO adBidRankNewPid(BidRankRecommendRequestParam request) {
+
+        ScoreParam param= RequestConvert.requestConvert(request);
+        LocalDateTime currentTime = LocalDateTime.now();
+        int currentHour = currentTime.getHour();
+        int dayOfWeek = currentTime.getDayOfWeek().getValue();
+        param.getRequestContext().setHour(currentHour+"");
+        param.getRequestContext().setWeek(dayOfWeek+"");
+        param.getRequestContext().setRegion(request.getRegion().replace("省",""));
+        param.getRequestContext().setCity(request.getCity().replace("市",""));
+        param.getRequestContext().setDay(currentTime.format(dateFormatter));
+
+        UserAdFeature userAdFeature=featureRemoteService.getUserAdFeature(request.getMid());
+        if(userAdFeature==null){
+            userAdFeature=new UserAdFeature();
+        }
+        Map<String,List<AdPlatformBidCreativeDTO>> groupMap=request
+                .getCreativeList()
+                .stream()
+                .collect(Collectors.groupingBy(creativeDTO -> creativeDTO.getCreativeId()+""));
 
+        List<AdRankItem> rankItems=featureRemoteService.
+                getAllAdFeatureList(
+                        new ArrayList<>(groupMap.keySet())
+                );
+
+        for(AdRankItem item:rankItems){
+            try {
+                AdPlatformBidCreativeDTO dto=groupMap.get(item.getAdId()+"").get(0);
+                item.setBid1(dto.getBid1());
+                item.setBid2(dto.getBid2());
+                item.setCpa(dto.getCpa());
+                item.setPidLambda(PidLambdaV2Container.getPidLambda(item.getAdId()));
+            }catch (Exception e){
+                e.printStackTrace();
+            }
+        }
+        List<AdRankItem> rankResult;
+        if(rankItems==null|| rankItems.size()==0){
+            rankItems=new LinkedList<>();
+            for(AdPlatformBidCreativeDTO dto:request.getCreativeList()){
+                AdRankItem item=new AdRankItem();
+                item.setBid1(dto.getBid1());
+                item.setBid2(dto.getBid2());
+                item.setCpa(dto.getCpa());
+                item.setAdId(dto.getCreativeId());
+                item.setItemFeature(new AdItemFeature());
+                item.setPidLambda(PidLambdaV2Container.getPidLambda(item.getAdId()));
+                rankItems.add(item);
+            }
+            rankResult=rankServiceThompson.rank(param, userAdFeature, rankItems,null);
+        }else {
+            rankResult=rank(param, userAdFeature, rankItems,ScorerUtils.BASE_CONF);
+        }
+
+        AdRankItem topItem=rankResult.get(0);
+
+        AdPlatformBidCreativeDTO result=AdPlatformBidCreativeDTO
+                .builder()
+                .build();
+        BeanUtils.copyProperties(topItem,result);
+        result.setCreativeId(topItem.getAdId());
+        result.setPctr(topItem.getCtr());
+        result.setPcvr(topItem.getCvr());
+        result.setCreativeCode(groupMap.get(topItem.getAdId()+"").get(0).getCreativeCode());
+        double realECpm=0d;
+        //经验值 待定
+        realECpm=topItem.getEcpm1();
+
+        if(realECpm>cpmMax/1000d){
+            realECpm=cpmMax/1000d;
+        }
+        if(realECpm<cpmMin/1000d){
+            realECpm=cpmMin/1000d;
+        }
+        result.setEcpm2(realECpm);
+        JSONObject object=new JSONObject();
+        object.put("mid",request.getMid());
+        object.put("adid",result.getCreativeId());
+        object.put("creativeCode",result.getCreativeCode());
+        object.put("type",topItem.getScore_type());
+        object.put("pctr",topItem.getCtr());
+        object.put("pcvr",topItem.getCvr());
+        object.put("lrsamples",topItem.getLrSampleString());
+        object.put("pidLambda",topItem.getPidLambda());
+        //临时加入供pid v2使用
+        object.put("realECpm",realECpm);
+        object.put("creativeId",result.getCreativeId());
+        object.put("cpa",topItem.getCpa());
+        object.put("dataTime",currentTime.format(timeFormatter));
+        log.info("svc=adBidRankNewPid {}", JSONObject.toJSONString(object));
+        object.remove("lrsamples");
+        log.info("svc=pid_v2_log obj={}", JSONObject.toJSONString(object));
+        return result;
+    }
 
     public List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems,String configFile){
         List<AdRankItem> rankResult = ScorerUtils

ad-engine-service/src/main/java/com/tzld/piaoquan/ad/engine/service/score/impl/RankServiceThompsonImpl.java

@@ -50,6 +50,11 @@ public class RankServiceThompsonImpl implements RankService {
         return null;
     }
 
+    @Override
+    public AdPlatformBidCreativeDTO adBidRankNewPid(BidRankRecommendRequestParam request) {
+        return null;
+    }
+
     public List<AdRankItem> rank (ScoreParam param, UserAdFeature userAdFeature, List<AdRankItem> rankItems,String configFile){
         if(configFile==null){
             configFile=ScorerUtils.THOMPSON_CONF;