video dssm model

recommend-model-produce/src/main/python/models/dssm/bq_reader_train_ps.py

@@ -4,47 +4,34 @@ import sys
 class DSSMReader(MultiSlotDataGenerator):
     def __init__(self):
         super(DSSMReader, self).__init__()
-        self.feature_dim = 3  # 设置特征维度
+        self.feature_dim = 5  # 设置特征维度
 
     def init(self, config=None):
         pass
 
     def line_process(self, line):
         try:
-            features = line.rstrip('\n').split('\t')
-            if len(features) < 3:  # 确保至少有query、pos_doc和一个neg_doc
-                return None
+            # 按tab分割样本的各个字段
+            sample_id, label, left_features, right_features = line.rstrip('\n').split('\t')
             
-            # 验证并处理query特征
-            query = features[0].split(',')
-            if len(query) != self.feature_dim:
-                return None
-            query = [float(x) for x in query]
-
-            # 验证并处理pos_doc特征
-            pos_doc = features[1].split(',')
-            if len(pos_doc) != self.feature_dim:
-                return None
-            pos_doc = [float(x) for x in pos_doc]
-
-            # 验证并处理neg_doc特征
-            neg_docs = []
-            for i in range(2, len(features)):
-                neg_doc = features[i].split(',')
-                if len(neg_doc) != self.feature_dim:
-                    continue
-                neg_docs.append([float(x) for x in neg_doc])
-
-            if not neg_docs:  # 如果没有有效的neg_doc
+            # 转换label为整数
+            label = int(label)
+            
+            # 处理左右视频特征
+            left_features = [float(x) for x in left_features.split(',')]
+            right_features = [float(x) for x in right_features.split(',')]
+            
+            # 验证特征维度
+            if len(left_features) != self.feature_dim or len(right_features) != self.feature_dim:
                 return None
-
+            
             # 构建输出列表
             output = []
-            output.append(("query", query))
-            output.append(("pos_doc", pos_doc))
-            for i, neg_doc in enumerate(neg_docs):
-                output.append((f"neg_doc_{i}", neg_doc))
-
+            output.append(("sample_id", [sample_id]))  # 样本ID
+            output.append(("label", [label]))          # 标签
+            output.append(("left_features", left_features))   # 左视频特征
+            output.append(("right_features", right_features)) # 右视频特征
+            
             return output
 
         except Exception as e:

recommend-model-produce/src/main/python/models/dssm/net.py

@@ -1,72 +1,141 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import paddle
 import paddle.nn as nn
 import paddle.nn.functional as F
 import numpy as np
 
-
 class DSSMLayer(nn.Layer):
-    def __init__(self, trigram_d, neg_num, slice_end, hidden_layers,
-                 hidden_acts):
+    def __init__(self, feature_num=5, embedding_dim=8, output_dim=16, 
+                 hidden_layers=[64, 32], hidden_acts=["relu", "relu"]):
         super(DSSMLayer, self).__init__()
-
-        self.hidden_layers = [trigram_d] + hidden_layers
+        
+        self.feature_num = feature_num
+        self.embedding_dim = embedding_dim
+        self.output_dim = output_dim
+        # 第一层的输入维度是所有特征的embedding拼接
+        self.hidden_layers = [feature_num * embedding_dim] + hidden_layers + [output_dim]
         self.hidden_acts = hidden_acts
-        self.slice_end = slice_end
 
-        self._query_layers = []
+        # 为每个特征创建embedding层
+        self.left_embeddings = nn.LayerList([
+            nn.Linear(
+                1, 
+                embedding_dim,
+                weight_attr=paddle.ParamAttr(
+                    initializer=paddle.nn.initializer.XavierNormal()
+                ),
+                bias_attr=paddle.ParamAttr(
+                    initializer=paddle.nn.initializer.Constant(value=0.0)
+                )
+            ) for _ in range(feature_num)
+        ])
+
+        self.right_embeddings = nn.LayerList([
+            nn.Linear(
+                1, 
+                embedding_dim,
+                weight_attr=paddle.ParamAttr(
+                    initializer=paddle.nn.initializer.XavierNormal()
+                ),
+                bias_attr=paddle.ParamAttr(
+                    initializer=paddle.nn.initializer.Constant(value=0.0)
+                )
+            ) for _ in range(feature_num)
+        ])
+
+        # 左视频塔
+        self._left_tower = []
         for i in range(len(self.hidden_layers) - 1):
             linear = paddle.nn.Linear(
                 in_features=self.hidden_layers[i],
                 out_features=self.hidden_layers[i + 1],
                 weight_attr=paddle.ParamAttr(
-                    initializer=paddle.nn.initializer.XavierNormal(
-                        fan_in=self.hidden_layers[i],
-                        fan_out=self.hidden_layers[i + 1])),
+                    initializer=paddle.nn.initializer.XavierNormal()
+                ),
                 bias_attr=paddle.ParamAttr(
-                    initializer=paddle.nn.initializer.XavierNormal(
-                        fan_in=self.hidden_layers[i],
-                        fan_out=self.hidden_layers[i + 1])))
-            self.add_sublayer('query_linear_%d' % i, linear)
-            self._query_layers.append(linear)
-            if self.hidden_acts[i] == "relu":
+                    initializer=paddle.nn.initializer.Constant(value=0.0)
+                )
+            )
+            self.add_sublayer('left_linear_%d' % i, linear)
+            self._left_tower.append(linear)
+            
+            if i < len(hidden_acts) and self.hidden_acts[i] == "relu":
                 act = paddle.nn.ReLU()
-                self.add_sublayer('query_act_%d' % i, act)
-                self._query_layers.append(act)
+                self.add_sublayer('left_act_%d' % i, act)
+                self._left_tower.append(act)
 
-        self._doc_layers = []
+        # 右视频塔
+        self._right_tower = []
         for i in range(len(self.hidden_layers) - 1):
             linear = paddle.nn.Linear(
                 in_features=self.hidden_layers[i],
                 out_features=self.hidden_layers[i + 1],
                 weight_attr=paddle.ParamAttr(
-                    initializer=paddle.nn.initializer.XavierNormal(
-                        fan_in=self.hidden_layers[i],
-                        fan_out=self.hidden_layers[i + 1])),
+                    initializer=paddle.nn.initializer.XavierNormal()
+                ),
                 bias_attr=paddle.ParamAttr(
-                    initializer=paddle.nn.initializer.XavierNormal(
-                        fan_in=self.hidden_layers[i],
-                        fan_out=self.hidden_layers[i + 1])))
-            self.add_sublayer('pos_linear_%d' % i, linear)
-            self._doc_layers.append(linear)
-            if self.hidden_acts[i] == "relu":
+                    initializer=paddle.nn.initializer.Constant(value=0.0)
+                )
+            )
+            self.add_sublayer('right_linear_%d' % i, linear)
+            self._right_tower.append(linear)
+            
+            if i < len(hidden_acts) and self.hidden_acts[i] == "relu":
                 act = paddle.nn.ReLU()
-                self.add_sublayer('pos_act_%d' % i, act)
-                self._doc_layers.append(act)
+                self.add_sublayer('right_act_%d' % i, act)
+                self._right_tower.append(act)
+
+    def _process_features(self, features, embeddings):
+        # 将每个特征转换为embedding
+        embedded_features = []
+        for i in range(self.feature_num):
+            feature = paddle.slice(
+                features, 
+                axes=[1], 
+                starts=[i], 
+                ends=[i+1]
+            )
+            embedded = embeddings[i](feature)
+            embedded_features.append(embedded)
+        
+        # 将所有embedding连接起来
+        return paddle.concat(embedded_features, axis=1)
+
+    def forward(self, left_features, right_features):
+        # 获取两个视频的特征表示
+        left_vec, right_vec = self.get_vectors(left_features, right_features)
+        
+        # 计算相似度
+        sim_score = F.cosine_similarity(
+            left_vec, 
+            right_vec, 
+            axis=1
+        ).reshape([-1, 1])
+
+        return sim_score, left_vec, right_vec
+
+    def get_vectors(self, left_features, right_features):
+        """获取两个视频的16维特征向量"""
+        # 处理左视频特征
+        left_embedded = self._process_features(left_features, self.left_embeddings)
+        left_vec = left_embedded
+        for layer in self._left_tower:
+            left_vec = layer(left_vec)
+        
+        # 处理右视频特征
+        right_embedded = self._process_features(right_features, self.right_embeddings)
+        right_vec = right_embedded
+        for layer in self._right_tower:
+            right_vec = layer(right_vec)
+            
+        # 确保输出是L2归一化的
+        left_vec = F.normalize(left_vec, p=2, axis=1)
+        right_vec = F.normalize(right_vec, p=2, axis=1)
+        
+        return left_vec, right_vec
+
 
+    
+    
     def forward(self, input_data, is_infer):
         query_fc = input_data[0]
         for n_layer in self._query_layers:

recommend-model-produce/src/main/python/models/dssm/static_model.py

@@ -1,22 +1,7 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import math
 import paddle
 from net import DSSMLayer
 
-
 class StaticModel():
     def __init__(self, config):
         self.cost = None
@@ -24,59 +9,96 @@ class StaticModel():
         self._init_hyper_parameters()
 
     def _init_hyper_parameters(self):
-        self.trigram_d = self.config.get("hyper_parameters.trigram_d")
-        self.neg_num = self.config.get("hyper_parameters.neg_num")
-        self.hidden_layers = self.config.get("hyper_parameters.fc_sizes")
-        self.hidden_acts = self.config.get("hyper_parameters.fc_acts")
-        self.learning_rate = self.config.get("hyper_parameters.learning_rate")
-        self.slice_end = self.config.get("hyper_parameters.slice_end")
-        self.learning_rate = self.config.get(
-            "hyper_parameters.optimizer.learning_rate")
+        # 修改超参数初始化
+        self.feature_num = self.config.get("hyper_parameters.feature_num", 5)
+        self.embedding_dim = self.config.get("hyper_parameters.embedding_dim", 8)
+        self.output_dim = self.config.get("hyper_parameters.output_dim", 16)
+        self.hidden_layers = self.config.get("hyper_parameters.hidden_layers", [64, 32])
+        self.hidden_acts = self.config.get("hyper_parameters.hidden_acts", ["relu", "relu"])
+        self.learning_rate = self.config.get("hyper_parameters.optimizer.learning_rate", 0.001)
+        self.margin = self.config.get("hyper_parameters.margin", 0.3)  # 用于损失函数的margin参数
 
     def create_feeds(self, is_infer=False):
-        query = paddle.static.data(
-            name="query", shape=[-1, self.trigram_d], dtype='float32')
-        self.prune_feed_vars = [query]
-
-        doc_pos = paddle.static.data(
-            name="doc_pos", shape=[-1, self.trigram_d], dtype='float32')
+        # 定义输入数据占位符
+        sample_id = paddle.static.data(
+            name="sample_id", shape=[-1, 1], dtype='int64')
+        
+        left_features = paddle.static.data(
+            name="left_features", shape=[-1, self.feature_num], dtype='float32')
+        
+        right_features = paddle.static.data(
+            name="right_features", shape=[-1, self.feature_num], dtype='float32')
 
-        if is_infer:
-            return [query, doc_pos]
+        feeds_list = [sample_id, left_features, right_features]
+        
+        if not is_infer:
+            label = paddle.static.data(
+                name="label", shape=[-1, 1], dtype='float32')
+            feeds_list.append(label)
 
-        doc_negs = [
-            paddle.static.data(
-                name="doc_neg_" + str(i),
-                shape=[-1, self.trigram_d],
-                dtype="float32") for i in range(self.neg_num)
-        ]
-        feeds_list = [query, doc_pos] + doc_negs
         return feeds_list
 
     def net(self, input, is_infer=False):
-        dssm_model = DSSMLayer(self.trigram_d, self.neg_num, self.slice_end,
-                               self.hidden_layers, self.hidden_acts)
-        R_Q_D_p, hit_prob = dssm_model.forward(input, is_infer)
+        # 创建模型实例
+        dssm_model = DSSMLayer(
+            feature_num=self.feature_num,
+            embedding_dim=self.embedding_dim,
+            output_dim=self.output_dim,
+            hidden_layers=self.hidden_layers,
+            hidden_acts=self.hidden_acts
+        )
 
-        self.inference_target_var = R_Q_D_p
-        self.prune_target_var = dssm_model.query_fc
-        self.train_dump_fields = [dssm_model.query_fc, R_Q_D_p]
-        self.train_dump_params = dssm_model.params
-        self.infer_dump_fields = [dssm_model.doc_pos_fc]
         if is_infer:
-            fetch_dict = {'query_doc_sim': R_Q_D_p}
+            sample_id, left_features, right_features = input
+        else:
+            sample_id, left_features, right_features, label = input
+
+        # 获取相似度和特征向量
+        sim_score, left_vec, right_vec = dssm_model(left_features, right_features)
+
+        self.inference_target_var = sim_score
+        self.left_vector = left_vec
+        self.right_vector = right_vec
+
+        if is_infer:
+            fetch_dict = {
+                'sample_id': sample_id,
+                'similarity': sim_score,
+                'left_vector': left_vec,
+                'right_vector': right_vec
+            }
             return fetch_dict
-        loss = -paddle.sum(paddle.log(hit_prob), axis=-1)
-        avg_cost = paddle.mean(x=loss)
-        # print(avg_cost)
+
+        # 计算损失
+        # 使用带margin的二元交叉熵损失
+        pos_mask = paddle.cast(label > 0.5, 'float32')
+        neg_mask = 1.0 - pos_mask
+        
+        positive_loss = -pos_mask * paddle.log(paddle.clip(sim_score, 1e-8, 1.0))
+        negative_loss = -neg_mask * paddle.log(paddle.clip(1 - sim_score + self.margin, 1e-8, 1.0))
+        
+        loss = positive_loss + negative_loss
+        avg_cost = paddle.mean(loss)
+        
         self._cost = avg_cost
-        fetch_dict = {'Loss': avg_cost}
+
+        # 计算accuracy
+        predictions = paddle.cast(sim_score > 0.5, 'float32')
+        accuracy = paddle.mean(paddle.cast(paddle.equal(predictions, label), 'float32'))
+
+        fetch_dict = {
+            'loss': avg_cost,
+            'accuracy': accuracy,
+            #'similarity': sim_score,
+            #'left_vector': left_vec,
+            #'right_vector': right_vec
+        }
         return fetch_dict
 
     def create_optimizer(self, strategy=None):
         optimizer = paddle.optimizer.Adam(
-            learning_rate=self.learning_rate, lazy_mode=True)
-        if strategy != None:
+            learning_rate=self.learning_rate)
+        if strategy is not None:
             import paddle.distributed.fleet as fleet
             optimizer = fleet.distributed_optimizer(optimizer, strategy)
         optimizer.minimize(self._cost)