Deep_match_rank_emb/deep_rank/FiBiNET_tzld_rank.py

import numpy as np
import pandas as pd
import gc
import os
import time
from sklearn.preprocessing import LabelEncoder
from tensorflow.python.keras.preprocessing.sequence import pad_sequences

from deepctr.feature_column import SparseFeat, VarLenSparseFeat, get_feature_names

import tensorflow as tf
from deepctr.models import *

def split(x):
    key_ans = x.split('|')
    for key in key_ans:
        if key not in key2index:
            # Notice : input value 0 is a special "padding",so we do not use 0 to encode valid feature for sequence input
            key2index[key] = len(key2index) + 1
    return list(map(lambda x: key2index[x], key_ans))


if __name__ == "__main__":

    begin_time = time.time()
    data = pd.read_csv("/work/xielixun/dwa_sum_graphembedding_user_action_feature_app_20210225.csv")
    sparse_features = ["videoid", "mid",
                       "videoGenre1", "videoGenre2",  "userRatedVideo1", "userRatedVideo2", "userRatedVideo3", "userGenre1", "userGenre2", "userCity",
                       "authorid", "userRealplayCount", "videoRealPlayCount", "videoDuration"]

    target = ['label']
    feature_max_idx = {}
    data = data[data["mid"] != "unknown"].copy()
    data["mid"].replace("unknown", "N000111111D", inplace=True)
    data = data[data["mid"] != "N000111111D"].copy()

    # 和上面函数的功能是一样的，见 deepMatch  DSSM
    def add_index_column(param_df, column_name):
        values = list(param_df[column_name].unique())
        value_index_dict = {value: idx for idx, value in enumerate(values)}
        if column_name == "mid":
            param_df["uidx"] = param_df[column_name].copy()
            param_df["mid"] = param_df[column_name].map(value_index_dict)
            feature_max_idx["mid"] = param_df["mid"].max() + 1

    add_index_column(data, "mid")

    for column_name in sparse_features:
        lbe = LabelEncoder()
        print("\n\n--------------   " + column_name)
        print(data[column_name])

        if column_name == "videoGenre1" or column_name == "videoGenre2" or \
                column_name == "videoGenre3" or column_name == "userGenre1" or column_name == "userGenre2" or column_name == "userGenre3":
            data[column_name].fillna("社会", inplace=True)

        if column_name == "userCity":
            data[column_name].fillna("北京", inplace=True)

        if column_name == "mid":
            continue

        data[column_name] = lbe.fit_transform(data[column_name])
        feature_max_idx[column_name] = data[column_name].max() + 1

    key2index = {}

    print("\n\n  ************ data process finish")
    user_video_list_df = data[data["label"] > 0].copy().groupby("mid")['videoid'].apply(list).reset_index()

    user_video_list_df.rename(columns={'videoid': 'hist_video_id'}, inplace=True)
    max_len = 50
    # print(list(user_video_length.keys()))
    # print(list(user_video_length.keys))
    mid_list = list(user_video_list_df["mid"])
    print(user_video_list_df["mid"])
    # print(mid_list)

    user_video_list_df["hist_len"] = user_video_list_df["hist_video_id"].apply(lambda x: len(x))
    print(user_video_list_df)
    print(len(user_video_list_df))
    emb_dim = 10

    fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique(), embedding_dim=emb_dim)
                              for feat in sparse_features]
    print(fixlen_feature_columns)

    use_weighted_sequence = False
    if use_weighted_sequence:
        varlen_feature_columns = [VarLenSparseFeat(SparseFeat('genres', vocabulary_size=len(
            key2index) + 1, embedding_dim=4), maxlen=max_len, combiner='mean',
                                                   weight_name='genres_weight')]  # Notice : value 0 is for padding for sequence input feature
    else:
        # varlen_feature_columns = [VarLenSparseFeat(SparseFeat('genres', vocabulary_size=len(
        #     key2index) + 1, embedding_dim=4), maxlen=max_len, combiner='mean',
        #                                            weight_name=None)]  # Notice : value 0 is for padding for sequence input feature
        varlen_feature_columns = [VarLenSparseFeat(SparseFeat('hist_video_id', vocabulary_size=feature_max_idx["videoid"], embedding_dim=emb_dim,
                                                              embedding_name="videoid"), maxlen=max_len, combiner='mean',
                                                   length_name="hist_len")]  # Notice : value 0 is for padding for sequence input feature

    linear_feature_columns = fixlen_feature_columns + varlen_feature_columns
    dnn_feature_columns = fixlen_feature_columns + varlen_feature_columns
    print(dnn_feature_columns)
    feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)
    df_merge = pd.merge(left=data,
                        right=user_video_list_df,
                        left_on="mid",
                        right_on="mid",
                        how="right")
    df_merge.head()

    print("df_merge len is:  ", len(df_merge))

    df_merge = df_merge.sample(frac=1.0)
    del data, user_video_list_df
    gc.collect()

    print("after sample df_merge len is:  ", len(df_merge))

    model_input = {name: df_merge[name] for name in sparse_features}  #
    video_hist_seq_pad = pad_sequences(df_merge["hist_video_id"], maxlen=max_len, padding='post', truncating='post', value=0)

    model_input["hist_video_id"] = video_hist_seq_pad
    print("\n\n\n")
    print(video_hist_seq_pad)
    print("\n\nuser_vids_input len is:  ", len(df_merge["hist_video_id"]))

    # model_input["genres_weight"] = np.random.randn(data.shape[0], max_len, 1)
    # model_input["hist_len"] = np.array(user_vids_len_input)
    model_input["hist_len"] = df_merge["hist_len"]

    print("\n\nuser_vids_len_input len is:  ", len(df_merge["hist_len"]))
    model = FiBiNET(linear_feature_columns, dnn_feature_columns, task='binary')
    logdir = os.path.join("log_callbacks")  # Tensorboard需要一个文件夹
    if not os.path.exists(logdir):
        os.mkdir(logdir)
    output_model_file = os.path.join(logdir,
                                     'xdeepfm_model.h5')

    callbacks = [
        tf.keras.callbacks.TensorBoard(logdir),
        tf.keras.callbacks.ModelCheckpoint(output_model_file,
                                           save_best_only=True),
        tf.keras.callbacks.EarlyStopping(patience=5, min_delta=1e-3),
    ]

    METRICS = [
        tf.keras.metrics.TruePositives(name='tp'),
        tf.keras.metrics.FalsePositives(name='fp'),
        tf.keras.metrics.TrueNegatives(name='tn'),
        tf.keras.metrics.FalseNegatives(name='fn'),
        tf.keras.metrics.BinaryAccuracy(name='accuracy'),
        tf.keras.metrics.Precision(name='precision'),
        tf.keras.metrics.Recall(name='recall'),
        # tf.keras.metrics.AUC(name='auc'),
        tf.keras.metrics.AUC(name='auc-ROC', curve='ROC'),
        tf.keras.metrics.AUC(name='auc-PRC', curve='PR')
    ]

    model.compile(
        loss='binary_crossentropy',
        optimizer='adam',
        metrics=METRICS
    )

    model.fit(model_input, df_merge[target].values,
                        batch_size=2048, epochs=5,
              verbose=2, validation_split=0.2, callbacks=callbacks)

    model.save("./tensorflow_xdeepfm-0325-tzld-2.h5")

    tf.keras.models.save_model(model,
                               "file:///work/xielixun/xDeepFM0325/tensorflow_xdeepfm-0325-tzld/001",
                               overwrite=True,
                               include_optimizer=True,
                               save_format=None,
                               signatures=None,
                               options=None)

    print("*******  train FiBiNET cost time is:  " + str(time.time() - begin_time))