支持视频生成接口

Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 
 COPY . /app
 
-RUN yarn install --registry https://registry.npmmirror.com/ && yarn run build
+RUN yarn install --registry https://registry.npmmirror.com/ --ignore-engines && yarn run build
 
 FROM node:lts-alpine
 

README.md

@@ -5,7 +5,7 @@
 ![](https://img.shields.io/github/forks/llm-red-team/glm-free-api.svg)
 ![](https://img.shields.io/docker/pulls/vinlic/glm-free-api.svg)
 
-支持高速流式输出、支持多轮对话、支持智能体对话、支持AI绘图、支持联网搜索、支持长文档解读、支持图像解析，零配置部署，多路token支持，自动清理会话痕迹。
+支持高速流式输出、支持多轮对话、支持智能体对话、支持视频生成、支持AI绘图、支持联网搜索、支持长文档解读、支持图像解析，零配置部署，多路token支持，自动清理会话痕迹。
 
 与ChatGPT接口完全兼容。
 
@@ -43,6 +43,7 @@ MiniMax（海螺AI）接口转API [hailuo-free-api](https://github.com/LLM-Red-T
 * [推荐使用客户端](#推荐使用客户端)
 * [接口列表](#接口列表)
   * [对话补全](#对话补全)
+  * [视频生成](#视频生成)
   * [AI绘图](#AI绘图)
   * [文档解读](#文档解读)
   * [图像解析](#图像解析)
@@ -92,6 +93,10 @@ https://udify.app/chat/Pe89TtaX3rKXM8NS
 
 ![多轮对话](./doc/example-6.png)
 
+### 视频生成Demo
+
+[点击预览](https://sfile.chatglm.cn/testpath/video/c1f59468-32fa-58c3-bd9d-ab4230cfe3ca_0.mp4)
+
 ### AI绘图Demo
 
 ![AI绘图](./doc/example-10.png)
@@ -322,9 +327,64 @@ Authorization: Bearer [refresh_token]
 }
 ```
 
+### 视频生成
+
+视频生成接口
+
+**如果您的账号未开通VIP，可能会因排队导致生成耗时较久**
+
+**POST /v1/videos/generations**
+
+header 需要设置 Authorization 头部：
+
+```
+Authorization: Bearer [refresh_token]
+```
+
+请求数据：
+```json
+{
+    // 模型名称
+    // cogvideox：默认官方视频模型
+    // cogvideox-pro：先生成图像再作为参考图像生成视频，作为视频首帧引导视频效果，但耗时更长
+    "model": "cogvideox",
+    // 视频生成提示词
+    "prompt": "一只可爱的猫走在花丛中",
+    // 支持使用图像URL或者BASE64_URL作为视频首帧参考图像（如果使用cogvideox-pro则会忽略此参数）
+    // "image_url": "https://sfile.chatglm.cn/testpath/b5341945-3839-522c-b4ab-a6268cb131d5_0.png",
+    // 支持设置视频风格：卡通3D/黑白老照片/油画/电影感
+    // "video_style": "油画",
+    // 支持设置情感氛围：温馨和谐/生动活泼/紧张刺激/凄凉寂寞
+    // "emotional_atmosphere": "生动活泼",
+    // 支持设置运镜方式：水平/垂直/推近/拉远
+    // "mirror_mode": "水平"
+}
+```
+
+响应数据：
+```json
+{
+    "created": 1722103836,
+    "data": [
+        {
+            // 对话ID，目前没啥用
+            "conversation_id": "66a537ec0603e53bccb8900a",
+            // 封面URL
+            "cover_url": "https://sfile.chatglm.cn/testpath/video_cover/c1f59468-32fa-58c3-bd9d-ab4230cfe3ca_cover_0.png",
+            // 视频URL
+            "video_url": "https://sfile.chatglm.cn/testpath/video/c1f59468-32fa-58c3-bd9d-ab4230cfe3ca_0.mp4",
+            // 视频时长
+            "video_duration": "6s",
+            // 视频分辨率
+            "resolution": "1440 × 960"
+        }
+    ]
+}
+```
+
 ### AI绘图
 
-对话补全接口，与openai的 [images-create-api](https://platform.openai.com/docs/api-reference/images/create) 兼容。
+图像生成接口，与openai的 [images-create-api](https://platform.openai.com/docs/api-reference/images/create) 兼容。
 
 **POST /v1/images/generations**
 

package.json

@@ -13,8 +13,8 @@
     "dist/"
   ],
   "scripts": {
-    "dev": "tsup src/index.ts --format cjs,esm --sourcemap --dts --publicDir public --watch --onSuccess \"node dist/index.js\"",
+    "dev": "tsup src/index.ts --format cjs,esm --sourcemap --dts --publicDir public --watch --onSuccess \"node --enable-source-maps dist/index.js\"",
-    "start": "node dist/index.js",
+    "start": "node --enable-source-maps dist/index.js",
     "build": "tsup src/index.ts --format cjs,esm --sourcemap --dts --clean --publicDir public"
   },
   "author": "Vinlic",
@@ -38,6 +38,7 @@
     "mime": "^4.0.1",
     "minimist": "^1.2.8",
     "randomstring": "^1.3.0",
+    "sharp": "^0.33.4",
     "uuid": "^9.0.1",
     "yaml": "^2.3.4"
   },

src/api/consts/exceptions.ts

@@ -7,5 +7,6 @@ export default {
     API_FILE_EXECEEDS_SIZE: [-2004, '远程文件超出大小'],
     API_CHAT_STREAM_PUSHING: [-2005, '已有对话流正在输出'],
     API_CONTENT_FILTERED: [-2006, '内容由于合规问题已被阻止生成'],
-    API_IMAGE_GENERATION_FAILED: [-2007, '图像生成失败']
+    API_IMAGE_GENERATION_FAILED: [-2007, '图像生成失败'],
+    API_VIDEO_GENERATION_FAILED: [-2008, '视频生成失败'],
 }

src/api/controllers/chat.ts

@@ -2,6 +2,8 @@ import { PassThrough } from "stream";
 import path from "path";
 import _ from "lodash";
 import mime from "mime";
+import sharp from "sharp";
+import fs from "fs-extra";
 import FormData from "form-data";
 import axios, { AxiosResponse } from "axios";
 
@@ -170,7 +172,7 @@ async function createCompletion(
   messages: any[],
   refreshToken: string,
   assistantId = DEFAULT_ASSISTANT_ID,
-  refConvId = '',
+  refConvId = "",
   retryCount = 0
 ) {
   return (async () => {
@@ -185,8 +187,7 @@ async function createCompletion(
       : [];
 
     // 如果引用对话ID不正确则重置引用
-    if (!/[0-9a-zA-Z]{24}/.test(refConvId))
+    if (!/[0-9a-zA-Z]{24}/.test(refConvId)) refConvId = "";
-      refConvId = '';
 
     // 请求流
     const token = await acquireToken(refreshToken);
@@ -221,7 +222,7 @@ async function createCompletion(
       }
     );
     if (result.headers["content-type"].indexOf("text/event-stream") == -1) {
-      result.data.on("data", buffer => logger.error(buffer.toString()));
+      result.data.on("data", (buffer) => logger.error(buffer.toString()));
       throw new APIException(
         EX.API_REQUEST_FAILED,
         `Stream response Content-Type invalid: ${result.headers["content-type"]}`
@@ -236,8 +237,8 @@ async function createCompletion(
     );
 
     // 异步移除会话
-    removeConversation(answer.id, refreshToken, assistantId).catch((err) =>
+    removeConversation(answer.id, refreshToken, assistantId).catch(
-      !refConvId && console.error(err)
+      (err) => !refConvId && console.error(err)
     );
 
     return answer;
@@ -272,7 +273,7 @@ async function createCompletionStream(
   messages: any[],
   refreshToken: string,
   assistantId = DEFAULT_ASSISTANT_ID,
-  refConvId = '',
+  refConvId = "",
   retryCount = 0
 ) {
   return (async () => {
@@ -287,8 +288,7 @@ async function createCompletionStream(
       : [];
 
     // 如果引用对话ID不正确则重置引用
-    if (!/[0-9a-zA-Z]{24}/.test(refConvId))
+    if (!/[0-9a-zA-Z]{24}/.test(refConvId)) refConvId = "";
-      refConvId = '';
 
     // 请求流
     const token = await acquireToken(refreshToken);
@@ -328,7 +328,7 @@ async function createCompletionStream(
         `Invalid response Content-Type:`,
         result.headers["content-type"]
       );
-      result.data.on("data", buffer => logger.error(buffer.toString()));
+      result.data.on("data", (buffer) => logger.error(buffer.toString()));
       const transStream = new PassThrough();
       transStream.end(
         `data: ${JSON.stringify({
@@ -359,8 +359,8 @@ async function createCompletionStream(
         `Stream has completed transfer ${util.timestamp() - streamStartTime}ms`
       );
       // 流传输结束后异步移除会话
-      removeConversation(convId, refreshToken, assistantId).catch((err) =>
+      removeConversation(convId, refreshToken, assistantId).catch(
-        !refConvId && console.error(err)
+        (err) => !refConvId && console.error(err)
       );
     });
   })().catch((err) => {
@@ -391,7 +391,10 @@ async function generateImages(
   return (async () => {
     logger.info(prompt);
     const messages = [
-      { role: "user", content: prompt.indexOf('画') == -1 ? `请画：${prompt}` : prompt },
+      {
+        role: "user",
+        content: prompt.indexOf("画") == -1 ? `请画：${prompt}` : prompt,
+      },
     ];
     // 请求流
     const token = await acquireToken(refreshToken);
@@ -458,6 +461,180 @@ async function generateImages(
   });
 }
 
+async function generateVideos(
+  model = "cogvideox",
+  prompt: string,
+  refreshToken: string,
+  options: {
+    imageUrl: string;
+    videoStyle: string;
+    emotionalAtmosphere: string;
+    mirrorMode: string;
+    audioId: string;
+  },
+  refConvId = "",
+  retryCount = 0
+) {
+  return (async () => {
+    logger.info(prompt);
+
+    // 如果引用对话ID不正确则重置引用
+    if (!/[0-9a-zA-Z]{24}/.test(refConvId)) refConvId = "";
+
+    const sourceList = [];
+    if (model == "cogvideox-pro") {
+      const imageUrls = await generateImages(undefined, prompt, refreshToken);
+      options.imageUrl = imageUrls[0];
+    }
+    if (options.imageUrl) {
+      const { source_id: sourceId } = await uploadFile(
+        options.imageUrl,
+        refreshToken,
+        true
+      );
+      sourceList.push(sourceId);
+    }
+
+    // 发起生成请求
+    let token = await acquireToken(refreshToken);
+    const result = await axios.post(
+      `https://chatglm.cn/chatglm/video-api/v1/chat`,
+      {
+        conversation_id: refConvId,
+        prompt,
+        source_list: sourceList.length > 0 ? sourceList : undefined,
+        advanced_parameter_extra: {
+          emotional_atmosphere: options.emotionalAtmosphere,
+          mirror_mode: options.mirrorMode,
+          video_style: options.videoStyle,
+        },
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Referer: "https://chatglm.cn/video",
+          "X-Device-Id": util.uuid(false),
+          "X-Request-Id": util.uuid(false),
+          ...FAKE_HEADERS,
+        },
+        // 30秒超时
+        timeout: 30000,
+        validateStatus: () => true,
+      }
+    );
+    const { result: _result } = checkResult(result, refreshToken);
+    const { chat_id: chatId, conversation_id: convId } = _result;
+
+    // 轮询生成进度
+    const startTime = util.unixTimestamp();
+    const results = [];
+    while (true) {
+      if (util.unixTimestamp() - startTime > 600)
+        throw new APIException(EX.API_VIDEO_GENERATION_FAILED);
+      const token = await acquireToken(refreshToken);
+      const result = await axios.get(
+        `https://chatglm.cn/chatglm/video-api/v1/chat/status/${chatId}`,
+        {
+          headers: {
+            Authorization: `Bearer ${token}`,
+            Referer: "https://chatglm.cn/video",
+            "X-Device-Id": util.uuid(false),
+            "X-Request-Id": util.uuid(false),
+            ...FAKE_HEADERS,
+          },
+          // 30秒超时
+          timeout: 30000,
+          validateStatus: () => true,
+        }
+      );
+      const { result: _result } = checkResult(result, refreshToken);
+      const {
+        status,
+        msg,
+        plan,
+        cover_url,
+        video_url,
+        video_duration,
+        resolution,
+      } = _result;
+      if (status != "init" && status != "processing") {
+        if (status != "finished")
+          throw new APIException(EX.API_VIDEO_GENERATION_FAILED);
+        let videoUrl = video_url;
+        if (options.audioId) {
+          const [key, id] = options.audioId.split("-");
+          const token = await acquireToken(refreshToken);
+          const result = await axios.post(
+            `https://chatglm.cn/chatglm/video-api/v1/static/composite_video`,
+            {
+              chat_id: chatId,
+              key,
+              audio_id: id,
+            },
+            {
+              headers: {
+                Authorization: `Bearer ${token}`,
+                Referer: "https://chatglm.cn/video",
+                "X-Device-Id": util.uuid(false),
+                "X-Request-Id": util.uuid(false),
+                ...FAKE_HEADERS,
+              },
+              // 30秒超时
+              timeout: 30000,
+              validateStatus: () => true,
+            }
+          );
+          const { result: _result } = checkResult(result, refreshToken);
+          videoUrl = _result.url;
+        }
+        results.push({
+          conversation_id: convId,
+          cover_url,
+          video_url: videoUrl,
+          video_duration,
+          resolution,
+        });
+        break;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
+
+    //https://chatglm.cn/chatglm/video-api/v1/reference/audio_group
+
+    axios
+      .delete(`https://chatglm.cn/chatglm/video-api/v1/chat/${chatId}`, {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Referer: "https://chatglm.cn/video",
+          "X-Device-Id": util.uuid(false),
+          "X-Request-Id": util.uuid(false),
+          ...FAKE_HEADERS,
+        },
+        validateStatus: () => true,
+      })
+      .catch((err) => logger.error("移除视频生成记录失败：", err));
+
+    return results;
+  })().catch((err) => {
+    if (retryCount < MAX_RETRY_COUNT) {
+      logger.error(`Video generation error: ${err.message}`);
+      logger.warn(`Try again after ${RETRY_DELAY / 1000}s...`);
+      return (async () => {
+        await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY));
+        return generateVideos(
+          model,
+          prompt,
+          refreshToken,
+          options,
+          refConvId,
+          retryCount + 1
+        );
+      })();
+    }
+    throw err;
+  });
+}
+
 /**
  * 提取消息中引用的文件URL
  *
@@ -508,24 +685,22 @@ function messagesPrepare(messages: any[], refs: any[], isRefConv = false) {
   if (isRefConv || messages.length < 2) {
     content = messages.reduce((content, message) => {
       if (_.isArray(message.content)) {
-        return (
+        return message.content.reduce((_content, v) => {
-          message.content.reduce((_content, v) => {
           if (!_.isObject(v) || v["type"] != "text") return _content;
           return _content + (v["text"] || "") + "\n";
-          }, content)
+        }, content);
-        );
       }
       return content + `${message.content}\n`;
     }, "");
     logger.info("\n透传内容：\n" + content);
-  }
+  } else {
-  else {
     // 检查最新消息是否含有"type": "image_url"或"type": "file",如果有则注入消息
     let latestMessage = messages[messages.length - 1];
     let hasFileOrImage =
       Array.isArray(latestMessage.content) &&
       latestMessage.content.some(
-        (v) => typeof v === "object" && ["file", "image_url"].includes(v["type"])
+        (v) =>
+          typeof v === "object" && ["file", "image_url"].includes(v["type"])
       );
     if (hasFileOrImage) {
       let newFileMessage = {
@@ -550,12 +725,10 @@ function messagesPrepare(messages: any[], refs: any[], isRefConv = false) {
           .replace("assistant", "<|assistant|>")
           .replace("user", "<|user|>");
         if (_.isArray(message.content)) {
-          return (
+          return message.content.reduce((_content, v) => {
-            message.content.reduce((_content, v) => {
             if (!_.isObject(v) || v["type"] != "text") return _content;
             return _content + (`${role}\n` + v["text"] || "") + "\n";
-            }, content)
+          }, content);
-          );
         }
         return (content += `${role}\n${message.content}\n`);
       }, "") + "<|assistant|>\n"
@@ -632,8 +805,13 @@ async function checkFileUrl(fileUrl: string) {
  *
  * @param fileUrl 文件URL
  * @param refreshToken 用于刷新access_token的refresh_token
+ * @param isVideoImage 是否是用于视频图像
  */
-async function uploadFile(fileUrl: string, refreshToken: string) {
+async function uploadFile(
+  fileUrl: string,
+  refreshToken: string,
+  isVideoImage: boolean = false
+) {
   // 预检查远程文件URL可用性
   await checkFileUrl(fileUrl);
 
@@ -660,6 +838,22 @@ async function uploadFile(fileUrl: string, refreshToken: string) {
   // 获取文件的MIME类型
   mimeType = mimeType || mime.getType(filename);
 
+  if (isVideoImage) {
+    const im = sharp(fileData).resize(1440, null, {
+      fit: "inside", // 保持宽高比
+    });
+    const metadata = await im.metadata();
+    const cropHeight = metadata.height > 960 ? 960 : metadata.height;
+    fileData = await im
+      .extract({
+        width: 1440,
+        height: cropHeight,
+        left: 0,
+        top: (metadata.height - cropHeight) / 2,
+      })
+      .toBuffer();
+  }
+
   const formData = new FormData();
   formData.append("file", fileData, {
     filename,
@@ -670,7 +864,9 @@ async function uploadFile(fileUrl: string, refreshToken: string) {
   const token = await acquireToken(refreshToken);
   let result = await axios.request({
     method: "POST",
-    url: "https://chatglm.cn/chatglm/backend-api/assistant/file_upload",
+    url: isVideoImage
+      ? "https://chatglm.cn/chatglm/video-api/v1/static/upload"
+      : "https://chatglm.cn/chatglm/backend-api/assistant/file_upload",
     data: formData,
     // 100M限制
     maxBodyLength: FILE_MAX_SIZE,
@@ -678,7 +874,9 @@ async function uploadFile(fileUrl: string, refreshToken: string) {
     timeout: 60000,
     headers: {
       Authorization: `Bearer ${token}`,
-      Referer: `https://chatglm.cn/`,
+      Referer: isVideoImage
+        ? "https://chatglm.cn/video"
+        : "https://chatglm.cn/",
       ...FAKE_HEADERS,
       ...formData.getHeaders(),
     },
@@ -731,7 +929,7 @@ async function receiveStream(stream: any): Promise<any> {
     let codeTemp = "";
     let lastExecutionOutput = "";
     let textOffset = 0;
-    let refContent = '';
+    let refContent = "";
     const parser = createParser((event) => {
       try {
         if (event.type !== "event") return;
@@ -835,7 +1033,13 @@ async function receiveStream(stream: any): Promise<any> {
           data.choices[0].message.content += chunk;
         } else {
           data.choices[0].message.content =
-            data.choices[0].message.content.replace(/【\d+†(来源|source)】/g, "") + (refContent ? `\n\n搜索结果来自：\n${refContent.replace(/\n$/, '')}` : '');
+            data.choices[0].message.content.replace(
+              /【\d+†(来源|源|source)】/g,
+              ""
+            ) +
+            (refContent
+              ? `\n\n搜索结果来自：\n${refContent.replace(/\n$/, "")}`
+              : "");
           resolve(data);
         }
       } catch (err) {
@@ -1082,16 +1286,12 @@ async function receiveImages(
                   imageUrls.push(value.image_url);
                 });
               }
-              if (
+              if (type == "text" && partStatus == "finish") {
-                type == "text" &&
-                partStatus == "finish"
-              ) {
                 const urlPattern = /\((https?:\/\/\S+)\)/g;
                 let match;
                 while ((match = urlPattern.exec(text)) !== null) {
                   const url = match[1];
-                  if (imageUrls.indexOf(url) == -1)
+                  if (imageUrls.indexOf(url) == -1) imageUrls.push(url);
-                    imageUrls.push(url);
                 }
               }
             });
@@ -1168,8 +1368,7 @@ async function getTokenLiveStatus(refreshToken: string) {
     const { result: _result } = checkResult(result, refreshToken);
     const { accessToken } = _result;
     return !!accessToken;
-  }
+  } catch (err) {
-  catch (err) {
     return false;
   }
 }
@@ -1178,6 +1377,7 @@ export default {
   createCompletion,
   createCompletionStream,
   generateImages,
+  generateVideos,
   getTokenLiveStatus,
   tokenSplit,
 };

src/api/routes/index.ts

@@ -3,6 +3,7 @@ import fs from 'fs-extra';
 import Response from '@/lib/response/Response.ts';
 import chat from "./chat.ts";
 import images from "./images.ts";
+import videos from './videos.ts';
 import ping from "./ping.ts";
 import token from './token.js';
 import models from './models.ts';
@@ -23,6 +24,7 @@ export default [
     },
     chat,
     images,
+    videos,
     ping,
     token,
     models

src/api/routes/videos.ts

@@ -0,0 +1,78 @@
+import _ from "lodash";
+
+import Request from "@/lib/request/Request.ts";
+import chat from "@/api/controllers/chat.ts";
+import util from "@/lib/util.ts";
+
+export default {
+
+  prefix: "/v1/videos",
+
+  post: {
+
+    "/generations": async (request: Request) => {
+      request
+        .validate(
+          "body.conversation_id",
+          (v) => _.isUndefined(v) || _.isString(v)
+        )
+        .validate("body.model", (v) => _.isUndefined(v) || _.isString(v))
+        .validate("body.prompt", _.isString)
+        .validate("body.audio_id", (v) => _.isUndefined(v) || _.isString(v))
+        .validate("body.image_url", (v) => _.isUndefined(v) || _.isString(v))
+        .validate(
+          "body.video_style",
+          (v) =>
+            _.isUndefined(v) ||
+            ["卡通3D", "黑白老照片", "油画", "电影感"].includes(v),
+          "video_style must be one of 卡通3D/黑白老照片/油画/电影感"
+        )
+        .validate(
+          "body.emotional_atmosphere",
+          (v) =>
+            _.isUndefined(v) ||
+            ["温馨和谐", "生动活泼", "紧张刺激", "凄凉寂寞"].includes(v),
+          "emotional_atmosphere must be one of 温馨和谐/生动活泼/紧张刺激/凄凉寂寞"
+        )
+        .validate(
+          "body.mirror_mode",
+          (v) =>
+            _.isUndefined(v) || ["水平", "垂直", "推近", "拉远"].includes(v),
+          "mirror_mode must be one of 水平/垂直/推近/拉远"
+        )
+        .validate("headers.authorization", _.isString);
+      // refresh_token切分
+      const tokens = chat.tokenSplit(request.headers.authorization);
+      // 随机挑选一个refresh_token
+      const token = _.sample(tokens);
+      const {
+        model,
+        conversation_id: convId,
+        prompt,
+        image_url: imageUrl,
+        video_style: videoStyle = "",
+        emotional_atmosphere: emotionalAtmosphere = "",
+        mirror_mode: mirrorMode = "",
+        audio_id: audioId,
+      } = request.body;
+      const data = await chat.generateVideos(
+        model,
+        prompt,
+        token,
+        {
+          imageUrl,
+          videoStyle,
+          emotionalAtmosphere,
+          mirrorMode,
+          audioId,
+        },
+        convId
+      );
+      return {
+        created: util.unixTimestamp(),
+        data,
+      };
+    },
+  },
+
+};

src/lib/request/Request.ts

@@ -52,7 +52,7 @@ export default class Request {
         this.time = Number(_.defaultTo(time, util.timestamp()));
     }
 
-    validate(key: string, fn?: Function) {
+    validate(key: string, fn?: Function, message?: string) {
         try {
             const value = _.get(this, key);
             if (fn) {
@@ -64,7 +64,7 @@ export default class Request {
         }
         catch (err) {
             logger.warn(`Params ${key} invalid:`, err);
-            throw new APIException(EX.API_REQUEST_PARAMS_INVALID, `Params ${key} invalid`);
+            throw new APIException(EX.API_REQUEST_PARAMS_INVALID, message || `Params ${key} invalid`);
         }
         return this;
     }