Просмотр исходного кода

Adjust streaming timeout for OpenAI models in OaiStreamHandler

- Implemented conditional logic to double the streaming timeout for models starting with "o1" or "o3".
- Improved handling of streaming timeout configuration to enhance performance based on model type.
1808837298@qq.com 1 год назад
Родитель
Сommit
8518ca65e2
1 измененных файлов с 6 добавлено и 2 удалено
  1. 6 2
      relay/channel/openai/relay-openai.go

+ 6 - 2
relay/channel/openai/relay-openai.go

@@ -65,8 +65,12 @@ func OaiStreamHandler(c *gin.Context, resp *http.Response, info *relaycommon.Rel
 	scanner.Split(bufio.ScanLines)
 
 	service.SetEventStreamHeaders(c)
-
-	ticker := time.NewTicker(time.Duration(constant.StreamingTimeout) * time.Second)
+	streamingTimeout := time.Duration(constant.StreamingTimeout) * time.Second
+	if strings.HasPrefix(info.UpstreamModelName, "o1") || strings.HasPrefix(info.UpstreamModelName, "o3") {
+		// twice timeout for o1 model
+		streamingTimeout *= 2
+	}
+	ticker := time.NewTicker(streamingTimeout)
 	defer ticker.Stop()
 
 	stopChan := make(chan bool)