Deep_match_rank_emb/deep_rank/WideNDeep_tzld_rank.py

# encoding: utf-8
import tensorflow as tf
import numpy as np
import pandas as pd
import gc
import os
import time
from sklearn.preprocessing import LabelEncoder
from tensorflow.python.keras.preprocessing.sequence import pad_sequences

begin_time = time.time()
data = pd.read_csv("/work/xielixun/dwa_sum_graphembedding_user_action_feature_app_20210225.csv")
sparse_features = ["videoid", "mid",
                   "videoGenre1", "videoGenre2",  "userRatedVideo1", "userRatedVideo2", "userRatedVideo3", "userGenre1", "userGenre2", "userCity",
                   "authorid", "userRealplayCount", "videoRealPlayCount", "videoDuration"]

target = ['label']
data = data.sort_values(by='logtimestamp', ascending=False)
# 需要把生成的label id  都保留下来，供后面线上使用，拼接特征
feature_max_idx = {}

data = data[data["mid"] != "unknown"].copy()
data["mid"].replace("unknown", "N000111111D", inplace=True)
data = data[data["mid"] != "N000111111D"].copy()

# 和上面函数的功能是一样的，见 deepMatch  DSSM
def add_index_column(param_df, column_name):
    values = list(param_df[column_name].unique())
    value_index_dict = {value: idx for idx, value in enumerate(values)}
    if column_name == "mid":
        param_df["uidx"] = param_df[column_name].copy()
        param_df["mid"] = param_df[column_name].map(value_index_dict) + 1
        feature_max_idx["mid"] = param_df["mid"].max() + 1

add_index_column(data, "mid")

for column_name in sparse_features:
    lbe = LabelEncoder()
    print("\n\n--------------   " + column_name)
    print(data[column_name])

    if column_name == "videoGenre1" or column_name == "videoGenre2" or \
            column_name == "videoGenre3" or column_name == "userGenre1" or column_name == "userGenre2" or column_name == "userGenre3":
        data[column_name].fillna("社会", inplace=True)
        continue

    if column_name == "userCity":
        data[column_name].fillna("北京", inplace=True)

    if column_name == "mid":
        continue
    if column_name == "videoid":
        data["vidx"] = data[column_name].copy()
        data["videoid"] = lbe.fit_transform(data[column_name]) + 1
        feature_max_idx["videoid"] = data["videoid"].max() + 1
        # print(data["videoid"])

    else:

        data[column_name] = lbe.fit_transform(data[column_name])
        feature_max_idx[column_name] = data[column_name].max() + 1


key2index = {}

print("\n\n  ************ data process finish")
user_video_list_df = data[data["label"] > 0].copy().groupby("mid")['videoid'].apply(list).reset_index()
user_video_list_df.rename(columns={'videoid': 'hist_video_id'}, inplace=True)
max_len = 50
mid_list = list(user_video_list_df["mid"])
user_video_list_df["hist_len"] = user_video_list_df["hist_video_id"].apply(lambda x: len(x))
emb_dim = 10
df_merge = pd.merge(left=data,
                    right=user_video_list_df,
                    left_on="mid",
                    right_on="mid",
                    how="right")
df_merge.head()
df_merge = df_merge.sample(frac=1.0)

g1 = data["videoGenre1"].values
g2 = data["videoGenre2"].values
g4 = data["userGenre1"].values
g5 = data["userGenre2"].values

del data, user_video_list_df
gc.collect()

model_input = {name: df_merge[name] for name in sparse_features}
video_hist_seq_pad = pad_sequences(df_merge["hist_video_id"], maxlen=max_len, padding='post', truncating='post', value=0)

model_input["hist_video_id"] = video_hist_seq_pad
model_input["hist_len"] = df_merge["hist_len"]

genre_set = g1 + g2 + g4 + g5
genre_vocab = list(set(genre_set))

GENRE_FEATURES = {
    'userGenre1': genre_vocab,
    'userGenre2': genre_vocab,
    'videoGenre1': genre_vocab,
    'videoGenre2': genre_vocab,
}

emb_dim1 = 10
emb_dim2 = 32

categorical_columns = []
for feature, vocab in GENRE_FEATURES.items():
    cat_col = tf.feature_column.categorical_column_with_vocabulary_list(
        key=feature, vocabulary_list=vocab)
    emb_col = tf.feature_column.embedding_column(cat_col, emb_dim1)
    categorical_columns.append(emb_col)

# video id embedding feature
video_col = tf.feature_column.categorical_column_with_identity(key='videoid', num_buckets=feature_max_idx["videoid"])
video_emb_col = tf.feature_column.embedding_column(video_col, emb_dim2)
categorical_columns.append(video_emb_col)

# user id embedding feature
user_col = tf.feature_column.categorical_column_with_identity(key='mid', num_buckets=feature_max_idx["mid"])
# user_emb_col = tf.feature_column.embedding_column(user_col, 10)
user_emb_col = tf.feature_column.embedding_column(user_col, emb_dim2)
categorical_columns.append(user_emb_col)

user_city_col = tf.feature_column.categorical_column_with_identity(key='userCity', num_buckets=feature_max_idx["userCity"])
user_city_emb_col = tf.feature_column.embedding_column(user_city_col, emb_dim1)
categorical_columns.append(user_city_emb_col)

authorid_col = tf.feature_column.categorical_column_with_identity(key='authorid', num_buckets=feature_max_idx["authorid"])
authorid_emb_col = tf.feature_column.embedding_column(authorid_col, emb_dim2)
categorical_columns.append(authorid_emb_col)

numerical_columns = [
                     tf.feature_column.numeric_column("userRealplayCount"),
                     tf.feature_column.numeric_column("videoRealPlayCount"),
                     tf.feature_column.numeric_column("videoDuration")
                    ]

# cross feature between current video and user historical video
rated_video1 = tf.feature_column.categorical_column_with_identity(key='userRatedVideo1', num_buckets=feature_max_idx["videoid"])
rated_video2 = tf.feature_column.categorical_column_with_identity(key='userRatedVideo2', num_buckets=feature_max_idx["videoid"])
rated_video3 = tf.feature_column.categorical_column_with_identity(key='userRatedVideo3', num_buckets=feature_max_idx["videoid"])

crossed_feature = tf.feature_column.indicator_column(tf.feature_column.crossed_column([video_col, rated_video1, rated_video2, rated_video3], feature_max_idx["videoid"]))

user_features = ["uidx", "mid",
                 "userRatedVideo1", "userRatedVideo2", "userRatedVideo3",
                 "userGenre1", "userGenre2", "userCity",
                 "userRealplayCount", "hist_video_id", "hist_len"]

item_features = ["vidx","videoid", "videoGenre1", "videoGenre2",
                 "authorid", "videoRealPlayCount", "videoDuration"]

df_merge["hist_video_id"] = list(map(lambda x: ' '.join(list(map(str, x))), video_hist_seq_pad))  # 上下两条功能相同

user_df = df_merge[user_features].drop_duplicates('uidx')
video_df = df_merge[item_features].drop_duplicates('vidx')
df_target = df_merge[target].copy()

del df_merge
gc.collect()

from datetime import datetime

TIMESTAMP = "{0:%Y-%m-%dT%H-%M-%S}".format(datetime.now())
user_df.to_csv("user_df" + TIMESTAMP + ".csv", index=False)
video_df.to_csv("video_df" + TIMESTAMP + ".csv", index=False)

del user_df, video_df
gc.collect()

inputs = {

    'videoid': tf.keras.layers.Input(name='videoid', shape=(), dtype='int32'),
    'mid': tf.keras.layers.Input(name='mid', shape=(), dtype='int32'),
    'userRatedVideo1': tf.keras.layers.Input(name='userRatedVideo1', shape=(), dtype='int32'),
    'userRatedVideo2': tf.keras.layers.Input(name='userRatedVideo2', shape=(), dtype='int32'),
    'userRatedVideo3': tf.keras.layers.Input(name='userRatedVideo3', shape=(), dtype='int32'),
    'userRealplayCount': tf.keras.layers.Input(name='userRealplayCount', shape=(), dtype='int32'),
    'videoRealPlayCount': tf.keras.layers.Input(name='videoRealPlayCount', shape=(), dtype='int32'),
    'videoDuration': tf.keras.layers.Input(name='videoDuration', shape=(), dtype='int32'),
    "authorid": tf.keras.layers.Input(name='authorid', shape=(), dtype='int32'),
    "userCity": tf.keras.layers.Input(name='userCity', shape=(), dtype='int32'),
    'userGenre1': tf.keras.layers.Input(name='userGenre1', shape=(), dtype='string'),
    'userGenre2': tf.keras.layers.Input(name='userGenre2', shape=(), dtype='string'),
    'videoGenre1': tf.keras.layers.Input(name='videoGenre1', shape=(), dtype='string'),
    'videoGenre2': tf.keras.layers.Input(name='videoGenre2', shape=(), dtype='string'),
    'hist_video_id':tf.keras.Input(shape=(50,), name='hist_video_id', dtype='int32'),
    'hist_len': tf.keras.Input(shape=(), name='hist_len', dtype='int32'),

}

from tensorflow.python.feature_column.feature_column import _LazyBuilder

hist_video_column = tf.feature_column.categorical_column_with_hash_bucket('hist_video_id', 2 * feature_max_idx["videoid"], dtype=tf.int32)
hist_video_embedded = tf.feature_column.embedding_column(hist_video_column, emb_dim2)
categorical_columns.append(hist_video_embedded)

deep = tf.keras.layers.DenseFeatures(numerical_columns + categorical_columns)(inputs)
deep = tf.keras.layers.Dense(128, activation='relu')(deep)
deep = tf.keras.layers.Dense(64, activation='relu')(deep)
# wide part for cross feature
wide = tf.keras.layers.DenseFeatures(crossed_feature)(inputs)
both = tf.keras.layers.concatenate([deep, wide])
output_layer = tf.keras.layers.Dense(1, activation='sigmoid')(both)
model = tf.keras.Model(inputs, output_layer)

logdir = os.path.join("log_callbacks")  # Tensorboard需要一个文件夹
if not os.path.exists(logdir):
    os.mkdir(logdir)
output_model_file = os.path.join(logdir,
                                 'wdl_model.h5')

callbacks = [
    tf.keras.callbacks.TensorBoard(logdir),
    tf.keras.callbacks.ModelCheckpoint(output_model_file,
                                       save_best_only=True),
    tf.keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3),
    ]

METRICS = [
    tf.keras.metrics.TruePositives(name='tp'),
    tf.keras.metrics.FalsePositives(name='fp'),
    tf.keras.metrics.TrueNegatives(name='tn'),
    tf.keras.metrics.FalseNegatives(name='fn'),
    tf.keras.metrics.BinaryAccuracy(name='accuracy'),
    tf.keras.metrics.Precision(name='precision'),
    tf.keras.metrics.Recall(name='recall'),
    tf.keras.metrics.AUC(name='auc'),
    tf.keras.metrics.AUC(name='auc-ROC', curve='ROC'),
    tf.keras.metrics.AUC(name='auc-PRC', curve='PR')
]


model.compile(
    loss='binary_crossentropy',
    optimizer='adam',
    metrics=METRICS
)

model.fit(model_input, df_target,
          batch_size=4096, epochs=1,
          verbose=2, validation_split=0.2, callbacks=callbacks)

model.save("./tensorflow_WDL-0416-tzld-2.h5")

tf.keras.models.save_model(model,
                           "file:///work/xielixun/WDL_0416/tensorflow_WDL-0416-tzld/001",
                           # "file:///Users/xielixun/Desktop/lixunxie/python_code/DeepMatch/examples/WDL_0416/tensorflow_WDL-0416-tzld/001",
                           overwrite=True,
                           include_optimizer=True,
                           save_format=None,
                           signatures=None,
                           options=None)

print("trainning wDL in 2021-0415 cost time:   " + str(time.time() - begin_time))