MigrateRepos/glm-free-api
1
0
Fork 0
mirror of https://github.com/LLM-Red-Team/glm-free-api.git synced 2024-12-22 00:49:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

支持视频生成接口

Browse Source
This commit is contained in:
Vinlic 2024-07-28 04:05:50 +08:00
parent d53a39f45a
commit f56e582ec6
9 changed files with 2325 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -4,7 +4,7 @@ WORKDIR /app
COPY . /app
RUN yarn install --registry https://registry.npmmirror.com/ && yarn run build
RUN yarn install --registry https://registry.npmmirror.com/ --ignore-engines && yarn run build
FROM node:lts-alpine

64
README.md
View File

@ -5,7 +5,7 @@
![](https://img.shields.io/github/forks/llm-red-team/glm-free-api.svg)
![](https://img.shields.io/docker/pulls/vinlic/glm-free-api.svg)
支持高速流式输出、支持多轮对话、支持智能体对话、支持AI绘图、支持联网搜索、支持长文档解读、支持图像解析，零配置部署，多路token支持，自动清理会话痕迹。
支持高速流式输出、支持多轮对话、支持智能体对话、支持视频生成、支持AI绘图、支持联网搜索、支持长文档解读、支持图像解析，零配置部署，多路token支持，自动清理会话痕迹。
与ChatGPT接口完全兼容。
@ -43,6 +43,7 @@ MiniMax（海螺AI）接口转API [hailuo-free-api](https://github.com/LLM-Red-T
* [推荐使用客户端](#推荐使用客户端)
* [接口列表](#接口列表)
* [对话补全](#对话补全)
* [视频生成](#视频生成)
* [AI绘图](#AI绘图)
* [文档解读](#文档解读)
* [图像解析](#图像解析)
@ -92,6 +93,10 @@ https://udify.app/chat/Pe89TtaX3rKXM8NS
![多轮对话](./doc/example-6.png)
### 视频生成Demo
[点击预览](https://sfile.chatglm.cn/testpath/video/c1f59468-32fa-58c3-bd9d-ab4230cfe3ca_0.mp4)
### AI绘图Demo
![AI绘图](./doc/example-10.png)
@ -322,9 +327,64 @@ Authorization: Bearer [refresh_token]
}
```
### 视频生成
视频生成接口
**如果您的账号未开通VIP，可能会因排队导致生成耗时较久**
**POST /v1/videos/generations**
header 需要设置 Authorization 头部：
```
Authorization: Bearer [refresh_token]
```
请求数据：
```json
{
// 模型名称
// cogvideox：默认官方视频模型
// cogvideox-pro：先生成图像再作为参考图像生成视频，作为视频首帧引导视频效果，但耗时更长
"model": "cogvideox",
// 视频生成提示词
"prompt": "一只可爱的猫走在花丛中",
// 支持使用图像URL或者BASE64_URL作为视频首帧参考图像（如果使用cogvideox-pro则会忽略此参数）
// "image_url": "https://sfile.chatglm.cn/testpath/b5341945-3839-522c-b4ab-a6268cb131d5_0.png",
// 支持设置视频风格：卡通3D/黑白老照片/油画/电影感
// "video_style": "油画",
// 支持设置情感氛围：温馨和谐/生动活泼/紧张刺激/凄凉寂寞
// "emotional_atmosphere": "生动活泼",
// 支持设置运镜方式：水平/垂直/推近/拉远
// "mirror_mode": "水平"
}
```
响应数据：
```json
{
"created": 1722103836,
"data": [
{
// 对话ID，目前没啥用
"conversation_id": "66a537ec0603e53bccb8900a",
// 封面URL
"cover_url": "https://sfile.chatglm.cn/testpath/video_cover/c1f59468-32fa-58c3-bd9d-ab4230cfe3ca_cover_0.png",
// 视频URL
"video_url": "https://sfile.chatglm.cn/testpath/video/c1f59468-32fa-58c3-bd9d-ab4230cfe3ca_0.mp4",
// 视频时长
"video_duration": "6s",
// 视频分辨率
"resolution": "1440 × 960"
}
]
}
```
### AI绘图
对话补全接口，与openai的 [images-create-api](https://platform.openai.com/docs/api-reference/images/create) 兼容。
图像生成接口，与openai的 [images-create-api](https://platform.openai.com/docs/api-reference/images/create) 兼容。
**POST /v1/images/generations**

5
package.json
View File

@ -13,8 +13,8 @@
"dist/"
],
"scripts": {
"dev": "tsup src/index.ts --format cjs,esm --sourcemap --dts --publicDir public --watch --onSuccess \"node dist/index.js\"",
"start": "node dist/index.js",
"dev": "tsup src/index.ts --format cjs,esm --sourcemap --dts --publicDir public --watch --onSuccess \"node --enable-source-maps dist/index.js\"",
"start": "node --enable-source-maps dist/index.js",
"build": "tsup src/index.ts --format cjs,esm --sourcemap --dts --clean --publicDir public"
},
"author": "Vinlic",
@ -38,6 +38,7 @@
"mime": "^4.0.1",
"minimist": "^1.2.8",
"randomstring": "^1.3.0",
"sharp": "^0.33.4",
"uuid": "^9.0.1",
"yaml": "^2.3.4"
},

3
src/api/consts/exceptions.ts
View File

@ -7,5 +7,6 @@ export default {
API_FILE_EXECEEDS_SIZE: [-2004, '远程文件超出大小'],
API_CHAT_STREAM_PUSHING: [-2005, '已有对话流正在输出'],
API_CONTENT_FILTERED: [-2006, '内容由于合规问题已被阻止生成'],
API_IMAGE_GENERATION_FAILED: [-2007, '图像生成失败']
API_IMAGE_GENERATION_FAILED: [-2007, '图像生成失败'],
API_VIDEO_GENERATION_FAILED: [-2008, '视频生成失败'],
}

314
src/api/controllers/chat.ts
View File

@ -2,6 +2,8 @@ import { PassThrough } from "stream";
import path from "path";
import _ from "lodash";
import mime from "mime";
import sharp from "sharp";
import fs from "fs-extra";
import FormData from "form-data";
import axios, { AxiosResponse } from "axios";
@ -170,7 +172,7 @@ async function createCompletion(
messages: any[],
refreshToken: string,
assistantId = DEFAULT_ASSISTANT_ID,
refConvId = '',
refConvId = "",
retryCount = 0
) {
return (async () => {
@ -180,13 +182,12 @@ async function createCompletion(
const refFileUrls = extractRefFileUrls(messages);
const refs = refFileUrls.length
? await Promise.all(
refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
)
refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
)
: [];
// 如果引用对话ID不正确则重置引用
if (!/[0-9a-zA-Z]{24}/.test(refConvId))
refConvId = '';
if (!/[0-9a-zA-Z]{24}/.test(refConvId)) refConvId = "";
// 请求流
const token = await acquireToken(refreshToken);
@ -221,7 +222,7 @@ async function createCompletion(
}
);
if (result.headers["content-type"].indexOf("text/event-stream") == -1) {
result.data.on("data", buffer => logger.error(buffer.toString()));
result.data.on("data", (buffer) => logger.error(buffer.toString()));
throw new APIException(
EX.API_REQUEST_FAILED,
`Stream response Content-Type invalid: ${result.headers["content-type"]}`
@ -236,8 +237,8 @@ async function createCompletion(
);
// 异步移除会话
removeConversation(answer.id, refreshToken, assistantId).catch((err) =>
!refConvId && console.error(err)
removeConversation(answer.id, refreshToken, assistantId).catch(
(err) => !refConvId && console.error(err)
);
return answer;
@ -272,7 +273,7 @@ async function createCompletionStream(
messages: any[],
refreshToken: string,
assistantId = DEFAULT_ASSISTANT_ID,
refConvId = '',
refConvId = "",
retryCount = 0
) {
return (async () => {
@ -282,13 +283,12 @@ async function createCompletionStream(
const refFileUrls = extractRefFileUrls(messages);
const refs = refFileUrls.length
? await Promise.all(
refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
)
refFileUrls.map((fileUrl) => uploadFile(fileUrl, refreshToken))
)
: [];
// 如果引用对话ID不正确则重置引用
if (!/[0-9a-zA-Z]{24}/.test(refConvId))
refConvId = '';
if (!/[0-9a-zA-Z]{24}/.test(refConvId)) refConvId = "";
// 请求流
const token = await acquireToken(refreshToken);
@ -328,7 +328,7 @@ async function createCompletionStream(
`Invalid response Content-Type:`,
result.headers["content-type"]
);
result.data.on("data", buffer => logger.error(buffer.toString()));
result.data.on("data", (buffer) => logger.error(buffer.toString()));
const transStream = new PassThrough();
transStream.end(
`data: ${JSON.stringify({
@ -359,8 +359,8 @@ async function createCompletionStream(
`Stream has completed transfer ${util.timestamp() - streamStartTime}ms`
);
// 流传输结束后异步移除会话
removeConversation(convId, refreshToken, assistantId).catch((err) =>
!refConvId && console.error(err)
removeConversation(convId, refreshToken, assistantId).catch(
(err) => !refConvId && console.error(err)
);
});
})().catch((err) => {
@ -391,7 +391,10 @@ async function generateImages(
return (async () => {
logger.info(prompt);
const messages = [
{ role: "user", content: prompt.indexOf('画') == -1 ? `请画：${prompt}` : prompt },
{
role: "user",
content: prompt.indexOf("画") == -1 ? `请画：${prompt}` : prompt,
},
];
// 请求流
const token = await acquireToken(refreshToken);
@ -458,6 +461,180 @@ async function generateImages(
});
}
async function generateVideos(
model = "cogvideox",
prompt: string,
refreshToken: string,
options: {
imageUrl: string;
videoStyle: string;
emotionalAtmosphere: string;
mirrorMode: string;
audioId: string;
},
refConvId = "",
retryCount = 0
) {
return (async () => {
logger.info(prompt);
// 如果引用对话ID不正确则重置引用
if (!/[0-9a-zA-Z]{24}/.test(refConvId)) refConvId = "";
const sourceList = [];
if (model == "cogvideox-pro") {
const imageUrls = await generateImages(undefined, prompt, refreshToken);
options.imageUrl = imageUrls[0];
}
if (options.imageUrl) {
const { source_id: sourceId } = await uploadFile(
options.imageUrl,
refreshToken,
true
);
sourceList.push(sourceId);
}
// 发起生成请求
let token = await acquireToken(refreshToken);
const result = await axios.post(
`https://chatglm.cn/chatglm/video-api/v1/chat`,
{
conversation_id: refConvId,
prompt,
source_list: sourceList.length > 0 ? sourceList : undefined,
advanced_parameter_extra: {
emotional_atmosphere: options.emotionalAtmosphere,
mirror_mode: options.mirrorMode,
video_style: options.videoStyle,
},
},
{
headers: {
Authorization: `Bearer ${token}`,
Referer: "https://chatglm.cn/video",
"X-Device-Id": util.uuid(false),
"X-Request-Id": util.uuid(false),
...FAKE_HEADERS,
},
// 30秒超时
timeout: 30000,
validateStatus: () => true,
}
);
const { result: _result } = checkResult(result, refreshToken);
const { chat_id: chatId, conversation_id: convId } = _result;
// 轮询生成进度
const startTime = util.unixTimestamp();
const results = [];
while (true) {
if (util.unixTimestamp() - startTime > 600)
throw new APIException(EX.API_VIDEO_GENERATION_FAILED);
const token = await acquireToken(refreshToken);
const result = await axios.get(
`https://chatglm.cn/chatglm/video-api/v1/chat/status/${chatId}`,
{
headers: {
Authorization: `Bearer ${token}`,
Referer: "https://chatglm.cn/video",
"X-Device-Id": util.uuid(false),
"X-Request-Id": util.uuid(false),
...FAKE_HEADERS,
},
// 30秒超时
timeout: 30000,
validateStatus: () => true,
}
);
const { result: _result } = checkResult(result, refreshToken);
const {
status,
msg,
plan,
cover_url,
video_url,
video_duration,
resolution,
} = _result;
if (status != "init" && status != "processing") {
if (status != "finished")
throw new APIException(EX.API_VIDEO_GENERATION_FAILED);
let videoUrl = video_url;
if (options.audioId) {
const [key, id] = options.audioId.split("-");
const token = await acquireToken(refreshToken);
const result = await axios.post(
`https://chatglm.cn/chatglm/video-api/v1/static/composite_video`,
{
chat_id: chatId,
key,
audio_id: id,
},
{
headers: {
Authorization: `Bearer ${token}`,
Referer: "https://chatglm.cn/video",
"X-Device-Id": util.uuid(false),
"X-Request-Id": util.uuid(false),
...FAKE_HEADERS,
},
// 30秒超时
timeout: 30000,
validateStatus: () => true,
}
);
const { result: _result } = checkResult(result, refreshToken);
videoUrl = _result.url;
}
results.push({
conversation_id: convId,
cover_url,
video_url: videoUrl,
video_duration,
resolution,
});
break;
}
await new Promise((resolve) => setTimeout(resolve, 1000));
}
//https://chatglm.cn/chatglm/video-api/v1/reference/audio_group
axios
.delete(`https://chatglm.cn/chatglm/video-api/v1/chat/${chatId}`, {
headers: {
Authorization: `Bearer ${token}`,
Referer: "https://chatglm.cn/video",
"X-Device-Id": util.uuid(false),
"X-Request-Id": util.uuid(false),
...FAKE_HEADERS,
},
validateStatus: () => true,
})
.catch((err) => logger.error("移除视频生成记录失败：", err));
return results;
})().catch((err) => {
if (retryCount < MAX_RETRY_COUNT) {
logger.error(`Video generation error: ${err.message}`);
logger.warn(`Try again after ${RETRY_DELAY / 1000}s...`);
return (async () => {
await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY));
return generateVideos(
model,
prompt,
refreshToken,
options,
refConvId,
retryCount + 1
);
})();
}
throw err;
});
}
/**
* 提取消息中引用的文件URL
*
@ -508,24 +685,22 @@ function messagesPrepare(messages: any[], refs: any[], isRefConv = false) {
if (isRefConv || messages.length < 2) {
content = messages.reduce((content, message) => {
if (_.isArray(message.content)) {
return (
message.content.reduce((_content, v) => {
if (!_.isObject(v) || v["type"] != "text") return _content;
return _content + (v["text"] || "") + "\n";
}, content)
);
return message.content.reduce((_content, v) => {
if (!_.isObject(v) || v["type"] != "text") return _content;
return _content + (v["text"] || "") + "\n";
}, content);
}
return content + `${message.content}\n`;
}, "");
logger.info("\n透传内容：\n" + content);
}
else {
} else {
// 检查最新消息是否含有"type": "image_url"或"type": "file",如果有则注入消息
let latestMessage = messages[messages.length - 1];
let hasFileOrImage =
Array.isArray(latestMessage.content) &&
latestMessage.content.some(
(v) => typeof v === "object" && ["file", "image_url"].includes(v["type"])
(v) =>
typeof v === "object" && ["file", "image_url"].includes(v["type"])
);
if (hasFileOrImage) {
let newFileMessage = {
@ -550,12 +725,10 @@ function messagesPrepare(messages: any[], refs: any[], isRefConv = false) {
.replace("assistant", "<|assistant|>")
.replace("user", "<|user|>");
if (_.isArray(message.content)) {
return (
message.content.reduce((_content, v) => {
if (!_.isObject(v) || v["type"] != "text") return _content;
return _content + (`${role}\n` + v["text"] || "") + "\n";
}, content)
);
return message.content.reduce((_content, v) => {
if (!_.isObject(v) || v["type"] != "text") return _content;
return _content + (`${role}\n` + v["text"] || "") + "\n";
}, content);
}
return (content += `${role}\n${message.content}\n`);
}, "") + "<|assistant|>\n"
@ -582,19 +755,19 @@ function messagesPrepare(messages: any[], refs: any[], isRefConv = false) {
...(fileRefs.length == 0
? []
: [
{
type: "file",
file: fileRefs,
},
]),
{
type: "file",
file: fileRefs,
},
]),
...(imageRefs.length == 0
? []
: [
{
type: "image",
image: imageRefs,
},
]),
{
type: "image",
image: imageRefs,
},
]),
],
},
];
@ -632,8 +805,13 @@ async function checkFileUrl(fileUrl: string) {
*
* @param fileUrl 文件URL
* @param refreshToken 用于刷新access_token的refresh_token
* @param isVideoImage 是否是用于视频图像
*/
async function uploadFile(fileUrl: string, refreshToken: string) {
async function uploadFile(
fileUrl: string,
refreshToken: string,
isVideoImage: boolean = false
) {
// 预检查远程文件URL可用性
await checkFileUrl(fileUrl);
@ -660,6 +838,22 @@ async function uploadFile(fileUrl: string, refreshToken: string) {
// 获取文件的MIME类型
mimeType = mimeType || mime.getType(filename);
if (isVideoImage) {
const im = sharp(fileData).resize(1440, null, {
fit: "inside", // 保持宽高比
});
const metadata = await im.metadata();
const cropHeight = metadata.height > 960 ? 960 : metadata.height;
fileData = await im
.extract({
width: 1440,
height: cropHeight,
left: 0,
top: (metadata.height - cropHeight) / 2,
})
.toBuffer();
}
const formData = new FormData();
formData.append("file", fileData, {
filename,
@ -670,7 +864,9 @@ async function uploadFile(fileUrl: string, refreshToken: string) {
const token = await acquireToken(refreshToken);
let result = await axios.request({
method: "POST",
url: "https://chatglm.cn/chatglm/backend-api/assistant/file_upload",
url: isVideoImage
? "https://chatglm.cn/chatglm/video-api/v1/static/upload"
: "https://chatglm.cn/chatglm/backend-api/assistant/file_upload",
data: formData,
// 100M限制
maxBodyLength: FILE_MAX_SIZE,
@ -678,7 +874,9 @@ async function uploadFile(fileUrl: string, refreshToken: string) {
timeout: 60000,
headers: {
Authorization: `Bearer ${token}`,
Referer: `https://chatglm.cn/`,
Referer: isVideoImage
? "https://chatglm.cn/video"
: "https://chatglm.cn/",
...FAKE_HEADERS,
...formData.getHeaders(),
},
@ -731,7 +929,7 @@ async function receiveStream(stream: any): Promise<any> {
let codeTemp = "";
let lastExecutionOutput = "";
let textOffset = 0;
let refContent = '';
let refContent = "";
const parser = createParser((event) => {
try {
if (event.type !== "event") return;
@ -835,7 +1033,13 @@ async function receiveStream(stream: any): Promise<any> {
data.choices[0].message.content += chunk;
} else {
data.choices[0].message.content =
data.choices[0].message.content.replace(/【\d+†(来源|source)】/g, "") + (refContent ? `\n\n搜索结果来自：\n${refContent.replace(/\n$/, '')}` : '');
data.choices[0].message.content.replace(
/【\d+†(来源|源|source)】/g,
""
) +
(refContent
? `\n\n搜索结果来自：\n${refContent.replace(/\n$/, "")}`
: "");
resolve(data);
}
} catch (err) {
@ -1008,8 +1212,8 @@ function createTransStream(stream: any, endCallback?: Function) {
index: 0,
delta:
result.status == "intervene" &&
result.last_error &&
result.last_error.intervene_text
result.last_error &&
result.last_error.intervene_text
? { content: `\n\n${result.last_error.intervene_text}` }
: {},
finish_reason: "stop",
@ -1082,16 +1286,12 @@ async function receiveImages(
imageUrls.push(value.image_url);
});
}
if (
type == "text" &&
partStatus == "finish"
) {
if (type == "text" && partStatus == "finish") {
const urlPattern = /\((https?:\/\/\S+)\)/g;
let match;
while ((match = urlPattern.exec(text)) !== null) {
const url = match[1];
if (imageUrls.indexOf(url) == -1)
imageUrls.push(url);
if (imageUrls.indexOf(url) == -1) imageUrls.push(url);
}
}
});
@ -1168,8 +1368,7 @@ async function getTokenLiveStatus(refreshToken: string) {
const { result: _result } = checkResult(result, refreshToken);
const { accessToken } = _result;
return !!accessToken;
}
catch (err) {
} catch (err) {
return false;
}
}
@ -1178,6 +1377,7 @@ export default {
createCompletion,
createCompletionStream,
generateImages,
generateVideos,
getTokenLiveStatus,
tokenSplit,
};

2
src/api/routes/index.ts
View File

@ -3,6 +3,7 @@ import fs from 'fs-extra';
import Response from '@/lib/response/Response.ts';
import chat from "./chat.ts";
import images from "./images.ts";
import videos from './videos.ts';
import ping from "./ping.ts";
import token from './token.js';
import models from './models.ts';
@ -23,6 +24,7 @@ export default [
},
chat,
images,
videos,
ping,
token,
models

78
src/api/routes/videos.ts Normal file
View File

@ -0,0 +1,78 @@
import _ from "lodash";
import Request from "@/lib/request/Request.ts";
import chat from "@/api/controllers/chat.ts";
import util from "@/lib/util.ts";
export default {
prefix: "/v1/videos",
post: {
"/generations": async (request: Request) => {
request
.validate(
"body.conversation_id",
(v) => _.isUndefined(v) || _.isString(v)
)
.validate("body.model", (v) => _.isUndefined(v) || _.isString(v))
.validate("body.prompt", _.isString)
.validate("body.audio_id", (v) => _.isUndefined(v) || _.isString(v))
.validate("body.image_url", (v) => _.isUndefined(v) || _.isString(v))
.validate(
"body.video_style",
(v) =>
_.isUndefined(v) ||
["卡通3D", "黑白老照片", "油画", "电影感"].includes(v),
"video_style must be one of 卡通3D/黑白老照片/油画/电影感"
)
.validate(
"body.emotional_atmosphere",
(v) =>
_.isUndefined(v) ||
["温馨和谐", "生动活泼", "紧张刺激", "凄凉寂寞"].includes(v),
"emotional_atmosphere must be one of 温馨和谐/生动活泼/紧张刺激/凄凉寂寞"
)
.validate(
"body.mirror_mode",
(v) =>
_.isUndefined(v) || ["水平", "垂直", "推近", "拉远"].includes(v),
"mirror_mode must be one of 水平/垂直/推近/拉远"
)
.validate("headers.authorization", _.isString);
// refresh_token切分
const tokens = chat.tokenSplit(request.headers.authorization);
// 随机挑选一个refresh_token
const token = _.sample(tokens);
const {
model,
conversation_id: convId,
prompt,
image_url: imageUrl,
video_style: videoStyle = "",
emotional_atmosphere: emotionalAtmosphere = "",
mirror_mode: mirrorMode = "",
audio_id: audioId,
} = request.body;
const data = await chat.generateVideos(
model,
prompt,
token,
{
imageUrl,
videoStyle,
emotionalAtmosphere,
mirrorMode,
audioId,
},
convId
);
return {
created: util.unixTimestamp(),
data,
};
},
},
};

4
src/lib/request/Request.ts
View File

@ -52,7 +52,7 @@ export default class Request {
this.time = Number(_.defaultTo(time, util.timestamp()));
}
validate(key: string, fn?: Function) {
validate(key: string, fn?: Function, message?: string) {
try {
const value = _.get(this, key);
if (fn) {
@ -64,7 +64,7 @@ export default class Request {
}
catch (err) {
logger.warn(`Params ${key} invalid:`, err);
throw new APIException(EX.API_REQUEST_PARAMS_INVALID, `Params ${key} invalid`);
throw new APIException(EX.API_REQUEST_PARAMS_INVALID, message || `Params ${key} invalid`);
}
return this;
}

1918
yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff