update

.dockerignore

@@ -0,0 +1,9 @@
+.git/
+.idea/
+.vscode/
+__pycache__/
+*.log
+*.jpg
+*.png
+*.gif
+*.webp

.gitignore

@@ -0,0 +1,318 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python,node
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,node
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+### Node Patch ###
+# Serverless Webpack directories
+.webpack/
+
+# Optional stylelint cache
+
+# SvelteKit build / generate output
+.svelte-kit
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+# VScode
+.vscode/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python,node
+
+.DS_Store

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY . .
+
+ENV TZ=Asia/Shanghai
+
+RUN mkdir -p /app/cache && pip install -r requirements.txt --no-cache-dir
+
+EXPOSE 8080
+
+ENTRYPOINT ["python", "/app/analyze_video.py"]

analyze_video.py

@@ -1,54 +1,27 @@
 import uvicorn
-import asyncio
 from fastapi import FastAPI
 from pydantic import BaseModel
 from google_ai.generativeai_video import main
-import google.generativeai as genai
 
 app = FastAPI()
-api_keys = [
-    {'key': 'AIzaSyB2kjF2-S2B5cJiosx_LpApd227w33CVvs', 'locked': False},
-    {'key': 'AIzaSyCor0q5w37Dy6fGxloLlCT7KqyEFU3PWP8', 'locked': False},
-]
-lock = asyncio.Lock()
+
 
 class VideoRequest(BaseModel):
+    api_key: str
     video_path: str
     prompt: str
     mark: str
     sample_data: str
 
-async def get_available_api_key():
-    """获取一个未锁定的 API key，如果没有可用的则等待 5 秒后重试"""
-    while True:
-        async with lock:
-            for key_data in api_keys:
-                if not key_data['locked']:
-                    key_data['locked'] = True  # 锁定该 key
-                    return key_data['key']
-        print( "没有可用的 API key，等待 5 秒后重试..." )
-        await asyncio.sleep( 5 )
-
-async def release_api_key(api_key):
-    """释放已锁定的 API key，并将其放到列表末尾"""
-    async with lock:
-        for i, key_data in enumerate( api_keys ):
-            if key_data['key'] == api_key:
-                key_data['locked'] = False  # 释放该 key
-                # 将释放的 key 移动到列表末尾
-                api_keys.append( api_keys.pop( i ) )
-                break
-
 
 @app.post("/process_video/")
 async def process_video(request: VideoRequest):
     """处理视频请求"""
-    global api_key_index  # 引用全局索引
+    api_key = request.api_key
     video_path = request.video_path
     prompt = request.prompt
     mark = request.mark
     sample_data = request.sample_data
-    api_key = await get_available_api_key()
     try:
         print("来一个请求，使用 API key:", api_key)
         result = await main(video_path, api_key, prompt, sample_data)
@@ -66,8 +39,6 @@ async def process_video(request: VideoRequest):
             "result": f"处理失败: {e}",
             "mark": f"处理失败: {e}"
         }
-    finally:
-        await release_api_key( api_key )
 
 
 if __name__ == "__main__":

docker-compose.yml

@@ -0,0 +1,42 @@
+version: '3.9'
+
+services:
+  server:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: google_ai_studio
+    container_name: google_server
+    restart: unless-stopped
+    environment:
+      - env=prod
+    networks:
+      - google_net
+  worker1:
+    depends_on:
+      - server
+    image: google_ai_studio
+    container_name: google_worker1
+    restart: unless-stopped
+    environment:
+      - env=prod
+      - API_KEY=AIzaSyB2kjF2-S2B5cJiosx_LpApd227w33CVvs
+    networks:
+      - google_net
+    entrypoint: "python /app/job_video_processing.py"
+  worker2:
+    depends_on:
+      - server
+    image: google_ai_studio
+    container_name: google_worker2
+    restart: unless-stopped
+    environment:
+      - env=prod
+      - API_KEY=AIzaSyCor0q5w37Dy6fGxloLlCT7KqyEFU3PWP8
+    networks:
+      - google_net
+    entrypoint: "python /app/job_video_processing.py"
+
+networks:
+  google_net:
+    name: google_net

google_ai/generativeai_video.py

@@ -8,6 +8,9 @@ import uuid
 from google.generativeai.types import HarmCategory, HarmBlockThreshold
 from common import Common
 
+env = os.getenv('env', 'dev')
+DIRECTORY = '/app/cache/' if env == 'prod' else os.path.expanduser('~/Downloads/')
+
 
 class VideoAnalyzer:
     def __init__(self, api_key):
@@ -24,7 +27,7 @@ class VideoAnalyzer:
         except Exception as e:
             print( f"处理或删除文件时发生错误: {str( e )}" )
 
-    async def download_video(self, video_url,  save_directory='/root/google_ai_studio/path'):
+    async def download_video(self, video_url,  save_directory=DIRECTORY):
     # async def download_video(self, video_url, save_directory='/Users/tzld/Desktop/google_ai_studio/path'):
 
         """从给定的视频链接下载视频并保存到指定路径"""

requirements.txt

@@ -0,0 +1,10 @@
+aliyun-log-python-sdk==0.9.12
+fastapi==0.115.3
+google-generativeai==0.8.3
+loguru==0.7.2
+odps==3.5.1
+opencv-python==4.10.0.84
+redis==5.1.1
+requests==2.32.3
+schedule==1.2.2
+uvicorn==0.32.0

video_processing/video_processing.py

@@ -1,4 +1,4 @@
-import re
+import os
 import time
 
 import requests
@@ -8,6 +8,9 @@ from common.aliyun_log import AliyunLogger
 from common.redis import get_video_data
 from common.feishu_data import Material
 
+HOST = 'google_server' if os.getenv('env', 'dev') == 'prod' else 'localhost'
+API_KEY = os.getenv('API_KEY')
+
 
 class VideoProcessing:
     def get_ai_data(self, video_path):
@@ -45,8 +48,9 @@ class VideoProcessing:
                 "二级分类": ["品类- 、分数-", "品类- 、分数-", "品类- 、分数-"]
             }
         }
-        url = "http://8.219.186.16:8080/process_video/"
+        url = f"http://{HOST}:8080/process_video/"
         payload = json.dumps( {
+            "api_key": API_KEY,
             "video_path": video_path,
             "prompt": prompt,
             "mark": str(mark),