import traceback
import requests
import json
import time
from datetime import datetime
# from db_helper import HologresHelper, RedisHelper, MysqlHelper
from db_helper import RedisHelper, MysqlHelper
from config import set_config
from log import Log
config_ = set_config()
log_ = Log()
def send_msg_to_feishu(msg_text):
"""发送消息到飞书"""
# webhook地址
webhook = 'https://open.feishu.cn/open-apis/bot/v2/hook/8de4de35-30ed-4692-8854-7a154e89b2f2'
# 自定义关键词key_word
key_word = '服务报警'
headers = {'Content-Type': 'application/json'}
payload_message = {
"msg_type": "text",
"content": {
"text": '{}: {}'.format(key_word, msg_text)
}
}
response = requests.request('POST', url=webhook, headers=headers, data=json.dumps(payload_message))
# print(response.text)
def request_post(request_url, request_data, timeout):
"""
post 请求 HTTP接口
:param request_url: 接口URL
:param request_data: 请求参数
:param timeout: 超时时间，单位为秒，type-float or tuple(connect_timeout, read_timeout)
:return: res_data json格式
"""
try:
headers = {"Connection": "close"}
response = requests.post(url=request_url, json=request_data, timeout=timeout, headers=headers)
if response.status_code == 200:
res_data = json.loads(response.text)
return res_data
else:
return None
except Exception as e:
log_.error('url: {}, exception: {}, traceback: {}'.format(request_url, e, traceback.format_exc()))
return None
def get_videos_remain_view_count(app_type, videos):
"""
获取视频在流量池中的剩余可分发数
:param app_type: 产品标识 type-int
:param videos: 视频信息 (视频id, 流量池标记) type-list，[{'videoId': video_id, 'flowPool': flow_pool}, ...]
:return: data type-list，[(video_id, flow_pool, view_count), ...]
error_flag 错误标记，True为错误
"""
error_flag = False
if not videos:
return [], error_flag
request_data = {'appType': app_type, 'videos': videos}
result = request_post(request_url=config_.GET_REMAIN_VIEW_COUNT_URL, request_data=request_data, timeout=(0.1, 1))
if result is None:
error_flag = True
return [], error_flag
if result['code'] != 0:
log_.info('获取视频在流量池中的剩余可分发数失败')
error_flag = True
return [], error_flag
data = [(item['videoId'], item['flowPool'], item['distributeCount']) for item in result['data']]
return data, error_flag
def get_videos_local_distribute_count(video_id, flow_pool):
"""
获取流量池视频本地分发数
:param video_id: video_id
:param flow_pool: 流量池标记
:return: current_count 本地记录的分发数
"""
# redis_h = datetime.now().hour
# if datetime.now().minute >= 30:
# redis_h += 0.5
# key_name = config_.LOCAL_DISTRIBUTE_COUNT_PREFIX + str(redis_h)
key_name = f'{config_.LOCAL_DISTRIBUTE_COUNT_PREFIX}{video_id}:{flow_pool}'
redis_helper = RedisHelper()
# video = '{}-{}'.format(video_id, flow_pool)
# current_count = redis_helper.get_score_with_value(key_name=key_name, value=video)
current_count = redis_helper.get_data_from_redis(key_name=key_name)
if current_count is not None:
return int(current_count)
else:
return None
def update_video_w_h_rate(video_id, key_name):
"""
获取横屏视频的宽高比，并存入redis中 （width/height>1)
:param video_id: videoId type-int
:param key_name: redis key
:return: None
"""
# 获取数据
sql = "SELECT id, width, height, rotate FROM longvideo.wx_video WHERE id = {};".format(video_id)
mysql_helper = MysqlHelper()
data = mysql_helper.get_data(sql=sql)
if len(data) == 0:
return
# 更新到redis
width, height, rotate = int(data[0][1]), int(data[0][2]), int(data[0][3])
if width == 0 or height == 0:
return
if rotate in (90, 270):
w_h_rate = height / width
else:
w_h_rate = width / height
if w_h_rate > 1:
info_data = {int(video_id): w_h_rate}
else:
return
redis_helper = RedisHelper()
# 写入新数据
if len(info_data) > 0:
redis_helper.add_data_with_zset(key_name=key_name, data=info_data)
class FilterVideos(object):
"""视频过滤"""
def __init__(self, request_id, app_type, video_ids, mid='', uid=''):
"""
初始化
:param request_id: request_id
:param app_type: 产品标识 type-int
:param video_ids: 需过滤的视频列表 type-list
:param mid: mid type-string
:param uid: uid type-string
"""
self.request_id = request_id
self.app_type = app_type
self.mid = mid
self.uid = uid
self.video_ids = video_ids
def filter_video_status_h(self, video_ids, rule_key, data_key, ab_code, province_code, key_flag=''):
"""召回小时级更新的视频状态过滤"""
# 根据Redis缓存中的数据过滤
redis_helper = RedisHelper()
# 获取不符合推荐状态的视频
if ab_code in [code for _, code in config_.AB_CODE['region_rank_by_h'].items()]:
if key_flag == 'region_24h':
key_prefix = f"{config_.REGION_H_VIDEO_FILER_24H}{province_code}."
elif key_flag == 'day_24h':
key_prefix = f"{config_.H_VIDEO_FILER_24H}{province_code}."
else:
key_prefix = f"{config_.REGION_H_VIDEO_FILER}{province_code}."
elif ab_code in [code for _, code in config_.AB_CODE['rank_by_24h'].items()]:
key_prefix = config_.H_VIDEO_FILER_24H
elif key_flag == '24h':
key_prefix = config_.H_VIDEO_FILER_24H
else:
key_prefix = config_.H_VIDEO_FILER
filter_videos_list = redis_helper.get_data_from_set(
key_name=f"{key_prefix}{self.app_type}.{data_key}.{rule_key}"
)
if not filter_videos_list:
return video_ids
filter_videos = [int(video) for video in filter_videos_list]
filtered_videos = [video_id for video_id in video_ids if video_id not in filter_videos]
return filtered_videos
def filter_videos_h(self, rule_key, data_key, ab_code, province_code, key_flag='', pool_type='rov'):
"""召回小时级更新的视频过滤"""
# 预曝光过滤
# st_pre = time.time()
filtered_pre_result = self.filter_video_previewed(self.video_ids)
# et_pre = time.time()
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': self.request_id,
# 'app_type': self.app_type,
# 'mid': self.mid,
# 'uid': self.uid,
# 'operation': 'preview_filter',
# 'request_videos': self.video_ids,
# 'preview_filter_result': filtered_pre_result,
# 'executeTime': (time.time() - st_pre) * 1000
# })
if not filtered_pre_result:
return None
# 视频状态过滤
# st_status = time.time()
filtered_status_result = self.filter_video_status_h(video_ids=filtered_pre_result, rule_key=rule_key,
data_key=data_key, ab_code=ab_code,
province_code=province_code, key_flag=key_flag)
# et_status = time.time()
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': self.request_id,
# 'app_type': self.app_type,
# 'mid': self.mid,
# 'uid': self.uid,
# 'operation': 'status_filter',
# 'request_videos': filtered_pre_result,
# 'status_filter_result': filtered_status_result,
# 'executeTime': (time.time() - st_status) * 1000
# })
if not filtered_status_result:
return None
# 视频已曝光过滤
st_viewed = time.time()
filtered_viewed_result = self.filter_video_viewed(video_ids=filtered_status_result)
# et_viewed = time.time()
log_.info({
'logTimestamp': int(time.time() * 1000),
'pool_type': pool_type,
'request_id': self.request_id,
'app_type': self.app_type,
'mid': self.mid,
'uid': self.uid,
'operation': 'view_filter',
'request_videos': filtered_status_result,
'view_filter_result': filtered_viewed_result,
'executeTime': (time.time() - st_viewed) * 1000
})
if not filtered_viewed_result:
return None
else:
return [int(video_id) for video_id in filtered_viewed_result]
def filter_videos(self, pool_type='rov', province_code=None):
"""视频过滤"""
# 预曝光过滤
st_pre = time.time()
filtered_pre_result = self.filter_video_previewed(self.video_ids)
# et_pre = time.time()
# log_.info({
# 'logTimestamp': int(time.time() * 1000),
# 'request_id': self.request_id,
# 'app_type': self.app_type,
# 'mid': self.mid,
# 'uid': self.uid,
# 'operation': 'preview_filter',
# 'request_videos': self.video_ids,
# 'preview_filter_result': filtered_pre_result,
# 'executeTime': (time.time() - st_pre) * 1000
# })
if not filtered_pre_result:
return None
# 视频状态过滤采用离线定时过滤方案
# 视频状态过滤
# st_status = time.time()
# filtered_status_result = self.filter_video_status(video_ids=filtered_pre_result)
# et_status = time.time()
# log_.info('filter by video status: result = {}, execute time = {}ms'.format(
# filtered_status_result, (et_status - st_status) * 1000))
# if not filtered_status_result:
# return None
# 视频已曝光过滤
st_viewed = time.time()
filtered_viewed_result = self.filter_video_viewed(video_ids=filtered_pre_result)
# et_viewed = time.time()
log_.info({
'logTimestamp': int(time.time() * 1000),
'pool_type': pool_type,
'request_id': self.request_id,
'app_type': self.app_type,
'mid': self.mid,
'uid': self.uid,
'operation': 'view_filter',
'request_videos': filtered_pre_result,
'view_filter_result': filtered_viewed_result,
'executeTime': (time.time() - st_viewed) * 1000
})
if not filtered_viewed_result:
return None
filtered_viewed_videos = [int(video_id) for video_id in filtered_viewed_result]
if pool_type == 'flow':
# 流量池视频需过滤屏蔽视频
if province_code is None:
return filtered_viewed_videos
else:
shield_key_name_list = config_.SHIELD_CONFIG.get(province_code, None)
if shield_key_name_list is not None:
filtered_shield_video_ids = self.filter_shield_video(
video_ids=filtered_viewed_videos, shield_key_name_list=shield_key_name_list
)
log_.info({
'logTimestamp': int(time.time() * 1000),
'pool_type': pool_type,
'request_id': self.request_id,
'app_type': self.app_type,
'mid': self.mid,
'uid': self.uid,
'operation': 'shield_filter',
'request_videos': filtered_viewed_videos,
'shield_filter_result': filtered_shield_video_ids,
'executeTime': (time.time() - st_viewed) * 1000
})
return filtered_shield_video_ids
else:
return filtered_viewed_videos
else:
return filtered_viewed_videos
def filter_video_previewed(self, video_ids):
"""
预曝光过滤
:param video_ids: 需过滤的视频列表 type-list
:return: filtered_videos 过滤后的列表 type-list
"""
if not self.mid or self.mid == 'null':
# mid为空时，不做预曝光过滤
return video_ids
# 根据Redis缓存中的数据过滤
redis_helper = RedisHelper()
# key拼接
key_name = f"{config_.PREVIEW_KEY_PREFIX}{self.app_type}:{self.mid}"
pe_videos_list = redis_helper.get_data_from_set(key_name)
# log_.info('****app_type = {}, mid = {}, uid = {}, pe_videos_list = {}'.format(
# self.app_type, self.mid, self.uid, pe_videos_list))
# log_.info('****app_type = {}, mid = {}, uid = {}, video_ids = {}'.format(
# self.app_type, self.mid, self.uid, video_ids))
if not pe_videos_list:
return video_ids
pe_videos = [int(video) for video in pe_videos_list]
filtered_videos = [video_id for video_id in video_ids if video_id not in pe_videos]
return filtered_videos
# def filter_video_status(self, video_ids):
# """
# 对视频状态进行过滤
# :param video_ids: 视频id列表 type-list
# :return: filtered_videos
# """
# if len(video_ids) == 1:
# sql = "set hg_experimental_enable_shard_pruning=off; " \
# "SELECT video_id " \
# "FROM {} " \
# "WHERE audit_status = 5 " \
# "AND applet_rec_status IN (1, -6) " \
# "AND open_status = 1 " \
# "AND payment_status = 0 " \
# "AND encryption_status != 5 " \
# "AND transcoding_status = 3 " \
# "AND video_id IN ({});".format(config_.VIDEO_STATUS, video_ids[0])
# else:
# sql = "set hg_experimental_enable_shard_pruning=off; " \
# "SELECT video_id " \
# "FROM {} " \
# "WHERE audit_status = 5 " \
# "AND applet_rec_status IN (1, -6) " \
# "AND open_status = 1 " \
# "AND payment_status = 0 " \
# "AND encryption_status != 5 " \
# "AND transcoding_status = 3 " \
# "AND video_id IN {};".format(config_.VIDEO_STATUS, tuple(video_ids))
#
# hologres_helper = HologresHelper()
# data = hologres_helper.get_data(sql=sql)
# filtered_videos = [int(temp[0]) for temp in data]
# return filtered_videos
def filter_video_viewed(self, video_ids, types=(1, 6,)):
"""
调用后端接口过滤用户已观看视频
:param video_ids: 视频id列表 type-list
:param types: 过滤参数 type-tuple, 默认(1, ) 1-已观看 2-视频状态 3-是否进入老年人社区 4-话题状态 5-推荐状态 6-白名单过滤
:return: filtered_videos
"""
# 调用http接口
request_data = {"appType": self.app_type,
"mid": self.mid,
"uid": self.uid,
"types": list(types),
"videoIds": video_ids}
result = request_post(request_url=config_.VIDEO_FILTER_URL, request_data=request_data, timeout=(0.1, 1))
if result is None:
# log_.info('过滤失败，types: {}'.format(types))
return []
if result['code'] != 0:
# log_.info('过滤失败，types: {}'.format(types))
return []
filtered_videos = result['data']
return filtered_videos
def filter_shield_video(self, video_ids, shield_key_name_list):
"""
过滤屏蔽视频视频
:param video_ids: 需过滤的视频列表 type-list
:param shield_key_name_list: 过滤视频 redis-key
:return: filtered_videos 过滤后的列表 type-list
"""
if len(video_ids) == 0:
return video_ids
# 根据Redis缓存中的数据过滤
redis_helper = RedisHelper()
for shield_key_name in shield_key_name_list:
shield_videos_list = redis_helper.get_data_from_set(key_name=shield_key_name)
if not shield_videos_list:
continue
shield_videos = [int(video) for video in shield_videos_list]
video_ids = [int(video_id) for video_id in video_ids if int(video_id) not in shield_videos]
return video_ids
if __name__ == '__main__':
# filter_ = FilterVideos(app_type=1, mid='22', uid='www', video_ids=[1, 2, 3, 55])
# filter_.filter_videos()
# filter_.filter_video_status(video_ids=[1, 3, 5])
# videos = [{'videoId': 9034659, 'flowPool': '3#11#3#1637824188547'}, {'videoId': 9035052, 'flowPool': '3#11#3#1637824172827'}]
# res = get_videos_remain_view_count(4, videos)
# print(res)
# text = '测试 @李倩'
# send_msg_to_feishu(text)
# update_video_w_h_rate(video_id=113, key_name='')
request_url = "http://videotest-internal.yishihui.com/longvideoapi/dev/test/testTimeout"
headers = {"Connection": "close"}
response = requests.post(url=request_url, json={}, timeout=(0.1, 1), headers=headers)
print(response.status_code, response.text)