JCNC/LLMX
2
1
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(core): 新增多模态聊天功能
Some checks failed
CI/CD Pipeline / build-and-deploy (push) Failing after 2h4m17s
Details

Browse Source 
- 在 llmx-core 中添加了多模态聊天相关的数据结构和接口
- 在 llmx-impl-bailian 中实现了多模态聊天的适配器和服务
- 新增了多模态聊天的控制器和相关配置- 重构了原有的聊天请求和响应结构，支持多模态内容
This commit is contained in:
gewuyou 2025-05-08 21:47:00 +08:00
parent 383533eb35
commit 7d71f4d32d
55 changed files with 839 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitea/workflows/deploy.test.yml
View File

@ -79,7 +79,7 @@ jobs:
./gradlew jib --stacktrace --build-cache --info -Dorg.gradle.caching=true -Dorg.gradle.jvmargs="-Xmx2g -Xms2g -XX:MaxMetaspaceSize=1g" | tee build.log
echo "=== 镜像构建结果 ==="
docker images | grep ${{env.PROJECT_NAME}}
docker images | grep ${{ env.PROJECT_NAME }} || true
- name: 🛑 Stop Gradle Daemon
run: |
echo "停止Gradle守护进程..."

6
build.gradle.kts
View File

@ -23,6 +23,7 @@ allprojects {
set(ProjectFlags.USE_LLM_CORE_SPI, false)
set(ProjectFlags.USE_SPRING_CLOUD_BOM, false)
set(ProjectFlags.IS_ROOT_MODULE, false)
set(ProjectFlags.USE_SPRING_BOOT_BOM,false)
}
repositories {
mavenLocal()
@ -93,6 +94,11 @@ subprojects {
implementation(platform(libs.springCloudDependencies.bom))
}
}
if(project.getPropertyByBoolean(ProjectFlags.USE_SPRING_BOOT_BOM)){
dependencies {
implementation(platform(libs.springBootDependencies.bom))
}
}
}
val libs = rootProject.libs
apply {

2
buildSrc/src/main/kotlin/Modules.kt
View File

@ -14,6 +14,8 @@ object Modules {
object Core{
// llmx-core-spi模块的路径，用于定义核心功能的SPI
const val SPI = ":llmx-core:llmx-core-spi"
const val API = ":llmx-core:llmx-core-api"
const val COMMON = ":llmx-core:llmx-core-common"
}
}

1
buildSrc/src/main/kotlin/ProjectFlags.kt
View File

@ -1,5 +1,6 @@
object ProjectFlags {
const val USE_SPRING_BOOT_WEB = "useSpringBootWeb"
const val USE_SPRING_BOOT_BOM = "useSpringBootBom"
const val USE_SPRING_CLOUD_BOM = "useSpringCloudBom"
const val USE_LLM_CORE_SPI = "useLLMCoreSPI"
const val IS_ROOT_MODULE = "isRootModule"

5
gradle/libs.versions.toml
View File

@ -8,6 +8,7 @@ spring-cloud-starter-alibaba-nacos-discovery-version = "2023.0.1.0"
forgeBoot-version = "1.1.0-SNAPSHOT"
okHttp-version = "4.12.0"
jib-version = "3.4.2"
org-reactivestreams-reactiveStreams-version = "1.0.4"
[plugins]
# 应用 Java 插件，提供基本的 Java 代码编译和构建能力
java = { id = "java" }
@ -31,6 +32,7 @@ jibLocalPlugin = { id = "org.jcnc.llmx.plugin.jib" }
jib-gradlePlugin = { module = "com.google.cloud.tools.jib:com.google.cloud.tools.jib.gradle.plugin", version.ref = "jib-version" }
# bom
springCloudDependencies-bom = { group ="org.springframework.cloud",name = "spring-cloud-dependencies", version.ref = "spring-cloud-version" }
springBootDependencies-bom = { module = "org.springframework.boot:spring-boot-dependencies", version.ref = "spring-boot-version" }
# kotlinx
# 响应式协程库
kotlinx-coruntes-reactor = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-reactor" }
@ -41,6 +43,7 @@ aliyun-bailian = { group = "com.aliyun", name = "bailian20231229", version.ref =
# SrpingCloud
springCloudStarter-alibaba-nacos-discovery = { group = "com.alibaba.cloud", name = "spring-cloud-starter-alibaba-nacos-discovery", version.ref = "spring-cloud-starter-alibaba-nacos-discovery-version" }
springCloudStarter-loadbalancer = { group = "org.springframework.cloud", name = "spring-cloud-starter-loadbalancer" }
springCloudStarter-openfeign = { group = "org.springframework.cloud", name = "spring-cloud-starter-openfeign" }
# SpringBootStarter
springBootStarter-webflux = { group = "org.springframework.boot", name = "spring-boot-starter-webflux" }
@ -49,6 +52,8 @@ springBootStarter-test = { group = "org.springframework.boot", name = "spring-bo
junitPlatform-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" }
# org-reactivestreams
org-reactivestreams-reactiveStreams = { group = "org.reactivestreams", name = "reactive-streams" ,version.ref="org-reactivestreams-reactiveStreams-version"}
# OkHttp
okHttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okHttp-version" }
# forgeBoot

2
gradlew vendored
View File

@ -207,7 +207,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# * For gewuyou: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
set -- \

3
llmx-app/.gitattributes vendored Normal file
View File

@ -0,0 +1,3 @@
/gradlew text eol=lf
*.bat text eol=crlf
*.jar binary

40
llmx-app/.gitignore vendored Normal file
View File

@ -0,0 +1,40 @@
HELP.md
.gradle
build/
!gradle/wrapper/gradle-wrapper.jar
!**/src/main/**/build/
!**/src/test/**/build/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
out/
!**/src/main/**/out/
!**/src/test/**/out/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Kotlin ###
.kotlin

9
llmx-app/build.gradle.kts Normal file
View File

@ -0,0 +1,9 @@
dependencies {
}
/**
* 由于 Kotlin 插件引入时会自动添加依赖，但根项目不需要 Kotlin 依赖，因此需要排除 Kotlin 依赖
*/
configurations.implementation {
exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib")
}

3
llmx-app/llmx-app-multimodality/.gitattributes vendored Normal file
View File

@ -0,0 +1,3 @@
/gradlew text eol=lf
*.bat text eol=crlf
*.jar binary

40
llmx-app/llmx-app-multimodality/.gitignore vendored Normal file
View File

@ -0,0 +1,40 @@
HELP.md
.gradle
build/
!gradle/wrapper/gradle-wrapper.jar
!**/src/main/**/build/
!**/src/test/**/build/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
out/
!**/src/main/**/out/
!**/src/test/**/out/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Kotlin ###
.kotlin

13
llmx-app/llmx-app-multimodality/build.gradle.kts Normal file
View File

@ -0,0 +1,13 @@
extra {
// 开启springboot
setProperty(ProjectFlags.USE_SPRING_BOOT_WEB, true)
setProperty(ProjectFlags.USE_SPRING_CLOUD_BOM, true)
}
dependencies {
// Nacos 服务发现和配置
implementation(libs.springCloudStarter.alibaba.nacos.discovery)
implementation(libs.forgeBoot.core.extension)
implementation(libs.forgeBoot.webmvc.version.springBootStarter)
implementation(libs.jackson.module.kotlin)
implementation(project(Modules.Core.API))
}

11
llmx-app/llmx-app-multimodality/src/main/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplication.kt Normal file
View File

@ -0,0 +1,11 @@
package org.jcnc.llmx.app.multimodality
import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.boot.runApplication
@SpringBootApplication
class LlmxAppMultimodalityApplication
fun main(args: Array<String>) {
runApplication<LlmxAppMultimodalityApplication>(*args)
}

8
llmx-app/llmx-app-multimodality/src/main/resources/application-dev.yml Normal file
View File

@ -0,0 +1,8 @@
server:
port: 8081
spring:
config:
import: classpath:bootstrap-dev.yml

0
llmx-core/llmx-core-service/src/main/resources/application-prod.yml → llmx-app/llmx-app-multimodality/src/main/resources/application-master.yml
View File

7
llmx-app/llmx-app-multimodality/src/main/resources/application-test.yml Normal file
View File

@ -0,0 +1,7 @@
server:
port: 9003
spring:
config:
import: classpath:bootstrap-test.yml

5
llmx-app/llmx-app-multimodality/src/main/resources/application.yml Normal file
View File

@ -0,0 +1,5 @@
spring:
application:
name: llmx-app-multimodality
profiles:
active: dev

12
llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-dev.yml Normal file
View File

@ -0,0 +1,12 @@
spring:
cloud:
nacos:
username: nacos
password: L4s6f9y3
server-addr: 49.235.96.75:8848
ip: 192.168.1.100
discovery:
server-addr: ${spring.cloud.nacos.server-addr}
username: ${spring.cloud.nacos.username}
password: ${spring.cloud.nacos.password}
ip: ${spring.cloud.nacos.ip}

10
llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-test.yml Normal file
View File

@ -0,0 +1,10 @@
spring:
cloud:
nacos:
username: nacos
password: L4s6f9y3
server-addr: 49.235.96.75:9001
discovery:
server-addr: ${spring.cloud.nacos.server-addr}
username: ${spring.cloud.nacos.username}
password: ${spring.cloud.nacos.password}

13
llmx-app/llmx-app-multimodality/src/test/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplicationTests.kt Normal file
View File

@ -0,0 +1,13 @@
package org.jcnc.llmx.app.multimodality
import org.junit.jupiter.api.Test
import org.springframework.boot.test.context.SpringBootTest
@SpringBootTest
class LlmxAppMultimodalityApplicationTests {
@Test
fun contextLoads() {
}
}

2
llmx-core/build.gradle.kts
View File

@ -2,7 +2,7 @@ dependencies {
}
/**
* 由于 Kotlin 插件被引入时会自动添加依赖，但根项目不需要 Kotlin 依赖，因此需要排除 Kotlin 依赖
* 由于 Kotlin 插件引入时会自动添加依赖，但根项目不需要 Kotlin 依赖，因此需要排除 Kotlin 依赖
*/
configurations.implementation {
exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib")

3
llmx-core/llmx-core-api/.gitattributes vendored Normal file
View File

@ -0,0 +1,3 @@
/gradlew text eol=lf
*.bat text eol=crlf
*.jar binary

40
llmx-core/llmx-core-api/.gitignore vendored Normal file
View File

@ -0,0 +1,40 @@
HELP.md
.gradle
build/
!gradle/wrapper/gradle-wrapper.jar
!**/src/main/**/build/
!**/src/test/**/build/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
out/
!**/src/main/**/out/
!**/src/test/**/out/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Kotlin ###
.kotlin

10
llmx-core/llmx-core-api/build.gradle.kts Normal file
View File

@ -0,0 +1,10 @@
ext{
set(ProjectFlags.USE_SPRING_CLOUD_BOM,true)
}
dependencies {
api(project(Modules.Core.COMMON))
api(libs.org.reactivestreams.reactiveStreams)
// Spring Cloud OpenFeign
implementation(libs.springCloudStarter.openfeign)
}

14
llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignAutoConfiguration.kt Normal file
View File

@ -0,0 +1,14 @@
package org.jcnc.llmx.core.api
import org.springframework.cloud.openfeign.EnableFeignClients
import org.springframework.context.annotation.Configuration
/**
*LLM 核心Feign自动配置
*
* @since 2025-05-08 16:48:23
* @author gewuyou
*/
@Configuration
@EnableFeignClients(basePackages = ["org.jcnc.llmx.core.api"])
open class LLMCoreFeignAutoConfiguration

21
llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignClient.kt Normal file
View File

@ -0,0 +1,21 @@
package org.jcnc.llmx.core.api
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.reactivestreams.Publisher
import org.springframework.cloud.openfeign.FeignClient
import org.springframework.http.MediaType
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
/**
*LLM 核心客户端
*
* @since 2025-05-08 16:22:22
* @author gewuyou
*/
@FeignClient(name = "llmx-core-service") // name 对应 Nacos 中注册的服务名
fun interface LLMCoreFeignClient {
@PostMapping("/api/v1/chat/stream", consumes = [MediaType.APPLICATION_JSON_VALUE])
fun chat(@RequestBody request: ChatRequest): Publisher<ChatResponsePart>
}

1
llmx-core/llmx-core-api/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports Normal file
View File

@ -0,0 +1 @@
org.jcnc.llmx.core.api.LLMCoreFeignAutoConfiguration

3
llmx-core/llmx-core-common/.gitattributes vendored Normal file
View File

@ -0,0 +1,3 @@
/gradlew text eol=lf
*.bat text eol=crlf
*.jar binary

40
llmx-core/llmx-core-common/.gitignore vendored Normal file
View File

@ -0,0 +1,40 @@
HELP.md
.gradle
build/
!gradle/wrapper/gradle-wrapper.jar
!**/src/main/**/build/
!**/src/test/**/build/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
out/
!**/src/main/**/out/
!**/src/test/**/out/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### Kotlin ###
.kotlin

7
llmx-core/llmx-core-common/build.gradle.kts Normal file
View File

@ -0,0 +1,7 @@
ext{
set(ProjectFlags.USE_SPRING_BOOT_BOM,true)
}
dependencies {
implementation(libs.jackson.core)
implementation(libs.jackson.module.kotlin)
}

42
llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/deserializer/MultiModalContentDeserializer.kt Normal file
View File

@ -0,0 +1,42 @@
package org.jcnc.llmx.core.common.deserializer
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.DeserializationContext
import com.fasterxml.jackson.databind.JsonDeserializer
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jcnc.llmx.core.common.entities.request.MultiModalContent
/**
* 多模态内容反序列化器
*
* 该类用于将JSON表示的多模态内容反序列化为MultiModalContent数据结构
* 主要处理两种情况：文本内容和图像内容，并相应地创建Text或Image实例
* 如果JSON中不包含已知的内容类型，则抛出IllegalArgumentException
*
* @since 2025-05-08 20:41:15
* @author gewuyou
*/
class MultiModalContentDeserializer : JsonDeserializer<MultiModalContent>() {
/**
* 反序列化方法
*
* 根据JSON节点中的内容类型，创建相应的MultiModalContent实例
* 如果内容类型未知，则抛出异常
*
* @param p JsonParser对象，用于解析JSON输入
* @param ctxt DeserializationContext对象，提供反序列化上下文
* @return MultiModalContent实例，表示反序列化的多模态内容
* @throws IllegalArgumentException 如果JSON节点中的内容类型未知
*/
override fun deserialize(p: JsonParser, ctxt: DeserializationContext): MultiModalContent {
// 解析JSON输入并将其读取为ObjectNode
val node: ObjectNode = p.codec.readTree(p)
// 根据JSON节点中的内容类型，返回相应的MultiModalContent实例
return when {
node.has("text") -> MultiModalContent.Text(node["text"].asText())
node.has("image") -> MultiModalContent.Image(node["image"].asText())
else -> throw IllegalArgumentException("Unknown content type in: $node")
}
}
}

52
llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/ChatRequest.kt Normal file
View File

@ -0,0 +1,52 @@
package org.jcnc.llmx.core.common.entities.request
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import org.jcnc.llmx.core.common.deserializer.MultiModalContentDeserializer
/**
* MultiModalContent密封类，用于定义消息内容的多种模态
* 这是一个密封类，意味着所有的子类都需要在这个文件中定义
*/
@JsonDeserialize(using = MultiModalContentDeserializer::class)
sealed class MultiModalContent {
/**
* Text数据类，表示文本内容的模态
* @param text 具体的文本内容
*/
data class Text(val text: String) : MultiModalContent()
/**
* Image数据类，表示图像内容的模态
* @param image 图像的base64
*/
data class Image(val image: String) : MultiModalContent()
}
/**
* ChatRequest数据类，用于封装聊天请求的参数
* @since 2025-04-25 17:07:02
* @author gewuyou
* @param prompt 用户的聊天提示或消息，是聊天请求的主要输入
* @param model 使用的聊天模型名称，决定了解析和响应的方式
* @param messages 包含多个模态消息的列表，每个消息有特定的角色和内容
* @param options 可选的额外参数集合，用于定制聊天请求的行为和输出
* 可以包括如最大回复长度、温度（随机性）等
*/
data class ChatRequest(
val prompt: String? = "",
val model: String,
val messages: List<MultiModalMessage> =listOf(),
val options: Map<String, String> = mapOf()
)
/**
* MultiModalMessage数据类，用于定义聊天请求中的单个消息
* @param role 消息的角色，可以是"system", "user", "assistant"等
* @param content 消息的内容，可以是文本、图像等多种模态的组合
*/
data class MultiModalMessage(
val role: String, // "system", "user", "assistant"
val content: List<MultiModalContent>
)

2
llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/EmbeddingRequest.kt → llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/EmbeddingRequest.kt
View File

@ -1,4 +1,4 @@
package org.jcnc.llmx.core.spi.entities.request
package org.jcnc.llmx.core.common.entities.request
/**
* 嵌入请求类

2
llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/ChatResponse.kt → llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/ChatResponse.kt
View File

@ -1,4 +1,4 @@
package org.jcnc.llmx.core.spi.entities.response
package org.jcnc.llmx.core.common.entities.response
/**
* 聊天响应类

2
llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/EmbeddingResponse.kt → llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/EmbeddingResponse.kt
View File

@ -1,4 +1,4 @@
package org.jcnc.llmx.core.spi.entities.response
package org.jcnc.llmx.core.common.entities.response
/**
* 嵌入响应类

25
llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/controller/ChatController.kt
View File

@ -1,10 +1,12 @@
package org.jcnc.llmx.core.service.controller
import com.gewuyou.forgeboot.core.extension.log
import com.gewuyou.forgeboot.webmvc.version.annotation.ApiVersion
import kotlinx.coroutines.flow.Flow
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.jcnc.llmx.core.service.service.impl.LLMServiceImpl
import org.jcnc.llmx.core.spi.entities.request.ChatRequest
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.reactivestreams.Publisher
import org.springframework.http.MediaType
import org.springframework.web.bind.annotation.PostMapping
@ -35,7 +37,22 @@ class ChatController(
* @return 返回一个Flow流，流中依次提供了聊天响应的部分数据，允许异步处理和逐步消费响应内容
*/
@PostMapping("/stream", produces = [MediaType.APPLICATION_NDJSON_VALUE])
fun chat(@RequestBody request: ChatRequest): Flow<ChatResponsePart> {
fun chat(@RequestBody request: ChatRequest): Publisher<ChatResponsePart> {
return llmServiceImpl.chat(request)
}
/**
* 处理多模态聊天请求的函数
* 该函数通过 POST 方法接收一个聊天请求，并以 NDJSON 的形式返回聊天响应的部分
*
* @param request 包含聊天请求信息的数据类
* @return 返回一个发布者，用于异步地发送聊天响应的部分
*
* 注意：该函数被设计为异步处理，以提高性能和响应性
* 它使用了响应式编程模型，适合处理高并发和大数据量的响应
*/
@PostMapping("/streamMultimodality", produces = [MediaType.APPLICATION_NDJSON_VALUE])
fun multimodalityChat(@RequestBody request: ChatRequest): Publisher<ChatResponsePart>{
log.info("request: {}", request)
return llmServiceImpl.multimodalityChat(request)
}
}

12
llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/LLMService.kt
View File

@ -1,8 +1,9 @@
package org.jcnc.llmx.core.service.service
import kotlinx.coroutines.flow.Flow
import org.jcnc.llmx.core.spi.entities.request.ChatRequest
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.reactivestreams.Publisher
import org.springframework.web.bind.annotation.PostMapping
@ -12,7 +13,7 @@ import org.springframework.web.bind.annotation.PostMapping
* @since 2025-04-26 17:38:18
* @author gewuyou
*/
fun interface LLMService {
interface LLMService {
/**
* 初始化与聊天服务的连接，以处理聊天请求
*
@ -23,5 +24,6 @@ fun interface LLMService {
* @return 返回一个Flow流，通过该流可以接收到聊天响应的部分数据，如消息、状态更新等
*/
@PostMapping("/chat")
fun chat(request: ChatRequest): Flow<ChatResponsePart>
fun chat(request: ChatRequest): Publisher<ChatResponsePart>
fun multimodalityChat(request: ChatRequest): Publisher<ChatResponsePart>
}

21
llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/impl/LLMServiceImpl.kt
View File

@ -1,11 +1,11 @@
package org.jcnc.llmx.core.service.service.impl
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.reactive.asFlow
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.jcnc.llmx.core.service.manager.ModelRouteManager
import org.jcnc.llmx.core.service.service.LLMService
import org.jcnc.llmx.core.spi.entities.request.ChatRequest
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.reactivestreams.Publisher
import org.springframework.stereotype.Service
import org.springframework.web.reactive.function.client.WebClient
@ -30,7 +30,7 @@ class LLMServiceImpl(
* @param request 聊天请求对象，包含建立聊天所需的信息，如用户标识、会话标识等
* @return 返回一个Flow流，通过该流可以接收到聊天响应的部分数据，如消息、状态更新等
*/
override fun chat(request: ChatRequest): Flow<ChatResponsePart> {
override fun chat(request: ChatRequest): Publisher<ChatResponsePart> {
val serviceName = modelRouteManager.resolveServiceName(request.model)
val webClient = webClientBuilder.build()
return webClient.post()
@ -38,6 +38,15 @@ class LLMServiceImpl(
.bodyValue(request)
.retrieve()
.bodyToFlux(ChatResponsePart::class.java)
.asFlow()
}
override fun multimodalityChat(request: ChatRequest): Publisher<ChatResponsePart> {
val serviceName = modelRouteManager.resolveServiceName(request.model)
val webClient = webClientBuilder.build()
return webClient.post()
.uri("http://$serviceName/provider/multimodalityChat")
.bodyValue(request)
.retrieve()
.bodyToFlux(ChatResponsePart::class.java)
}
}

1
llmx-core/llmx-core-service/src/main/resources/application-dev.yml
View File

@ -9,5 +9,6 @@ llmx:
qwen-turbo: llmx-impl-baiLian
qwen-max: llmx-impl-baiLian
qwen-plus: llmx-impl-baiLian
qwen-vl-max-latest: llmx-impl-baiLian

7
llmx-core/llmx-core-service/src/main/resources/application-master.yml Normal file
View File

@ -0,0 +1,7 @@
server:
port: 9002
spring:
cloud:
nacos:
discovery:
server-addr: 49.235.96.75:9001

1
llmx-core/llmx-core-service/src/main/resources/application-test.yml
View File

@ -9,5 +9,6 @@ llmx:
qwen-turbo: llmx-impl-baiLian
qwen-max: llmx-impl-baiLian
qwen-plus: llmx-impl-baiLian
qwen-vl-max-latest: llmx-impl-baiLian

3
llmx-core/llmx-core-spi/build.gradle.kts
View File

@ -4,6 +4,7 @@ apply {
plugin(libs.plugins.kotlin.plugin.spring.get().pluginId)
}
dependencies {
compileOnly(libs.kotlinx.coruntes.reactor)
compileOnly(libs.springBootStarter.web)
api(libs.org.reactivestreams.reactiveStreams)
api(project(Modules.Core.COMMON))
}

17
llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/ChatRequest.kt
View File

@ -1,17 +0,0 @@
package org.jcnc.llmx.core.spi.entities.request
/**
* ChatRequest数据类，用于封装聊天请求的参数
* @since 2025-04-25 17:07:02
* @author gewuyou
* @param prompt 用户的聊天提示或消息，是聊天请求的主要输入
* @param model 使用的聊天模型名称，决定了解析和响应的方式
* @param options 可选的额外参数集合，用于定制聊天请求的行为和输出
* 可以包括如最大回复长度、温度（随机性）等
*/
data class ChatRequest(
val prompt: String,
val model: String,
val options: Map<String, String> = mapOf()
)

22
llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/provider/LLMProvider.kt
View File

@ -1,9 +1,10 @@
package org.jcnc.llmx.core.spi.provider
import org.jcnc.llmx.core.spi.entities.request.ChatRequest
import org.jcnc.llmx.core.spi.entities.request.EmbeddingRequest
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.jcnc.llmx.core.spi.entities.response.EmbeddingResponse
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.request.EmbeddingRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.jcnc.llmx.core.common.entities.response.EmbeddingResponse
import org.reactivestreams.Publisher
import org.springframework.http.MediaType
import org.springframework.web.bind.annotation.PostMapping
@ -31,7 +32,18 @@ interface LLMProvider {
*/
@PostMapping("/chat", produces = [MediaType.APPLICATION_NDJSON_VALUE])
fun chat(request: ChatRequest): Publisher<ChatResponsePart>
/**
* 处理多模态聊天请求的函数
* 该函数通过 POST 方法接收一个聊天请求，并以 NDJSON 的形式返回聊天响应的部分
*
* @param request 包含聊天请求信息的数据类
* @return 返回一个发布者，用于异步地发送聊天响应的部分
*
* 注意：该函数被设计为异步处理，以提高性能和响应性
* 它使用了响应式编程模型，适合处理高并发和大数据量的响应
*/
@PostMapping("/multimodalityChat", produces = [MediaType.APPLICATION_NDJSON_VALUE])
fun multimodalityChat(request: ChatRequest): Publisher<ChatResponsePart>
/**
* 嵌入功能方法
* 该方法允许用户发送嵌入请求，以获取LLM生成的嵌入向量

10
llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/adapter/DashScopeAdapter.kt
View File

@ -11,7 +11,8 @@ import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.springframework.stereotype.Component
import java.io.BufferedReader
@ -30,7 +31,7 @@ import java.io.BufferedReader
@Component
class DashScopeAdapter(
private val okHttpClient: OkHttpClient,
private val objectMapper: ObjectMapper
private val objectMapper: ObjectMapper,
) {
/**
* 发送流式聊天请求
@ -50,7 +51,7 @@ class DashScopeAdapter(
headers: Map<String, String>,
requestBody: Any,
extractContent: (String) -> ChatResponsePart,
dispatcher: CoroutineDispatcher = Dispatchers.IO
dispatcher: CoroutineDispatcher = Dispatchers.IO,
): Flow<ChatResponsePart> = flow {
val requestJson = objectMapper.writeValueAsString(requestBody)
log.info("📤 请求参数: {}", requestJson)
@ -59,6 +60,7 @@ class DashScopeAdapter(
okHttpClient.newCall(request).execute().use { response ->
if (!response.isSuccessful) {
log.error("🚨 DashScope 请求失败: code ${response.code} message: ${response.message} body: ${response.body}")
throw RuntimeException("❌ DashScope 请求失败: HTTP ${response.code}")
}
@ -92,7 +94,7 @@ class DashScopeAdapter(
dispatcher: CoroutineDispatcher,
reader: BufferedReader,
extractContent: (String) -> ChatResponsePart,
allContent: StringBuilder
allContent: StringBuilder,
) {
while (currentCoroutineContext().isActive) {
val line = withContext(dispatcher) {

4
llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/config/entities/DashScopeProperties.kt
View File

@ -44,5 +44,9 @@ class DashScopeProperties {
* 应用调用基路径，例如 /api/v1。
*/
var baseUrl: String = ""
/**
* 多模态对话调用路径
*/
var multimodalityUrl: String = ""
}

13
llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/controller/BaiLianProvider.kt
View File

@ -2,10 +2,11 @@ package org.jcnc.llmx.impl.baiLian.controller
import kotlinx.coroutines.reactive.asPublisher
import org.jcnc.llmx.core.spi.entities.request.ChatRequest
import org.jcnc.llmx.core.spi.entities.request.EmbeddingRequest
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.jcnc.llmx.core.spi.entities.response.EmbeddingResponse
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.request.EmbeddingRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.jcnc.llmx.core.common.entities.response.EmbeddingResponse
import org.jcnc.llmx.core.spi.provider.LLMProvider
import org.jcnc.llmx.impl.baiLian.service.BaiLianModelService
import org.reactivestreams.Publisher
@ -37,6 +38,10 @@ class BaiLianProvider(
return baiLianModelService.streamChat(request).asPublisher()
}
override fun multimodalityChat(@RequestBody request: ChatRequest): Publisher<ChatResponsePart> {
return baiLianModelService.streamMultimodalityChat(request).asPublisher()
}
/**
* 嵌入功能方法
* 该方法允许用户发送嵌入请求，以获取LLM生成的嵌入向量

117
llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/entities/response/DashScopeResponse.kt Normal file
View File

@ -0,0 +1,117 @@
package org.jcnc.llmx.impl.baiLian.entities.response
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
/**
* Dash范围响应类
*
* 该类用于表示从Dash服务接收到的响应数据结构它包含了响应的输出信息、使用情况和请求ID
* 主要用于JSON序列化和反序列化
*
* @param output 输出信息，包括生成的内容和完成原因
* @param usage 使用情况，包括输入和输出的令牌详情
* @param requestId 请求ID，用于跟踪和调试请求
*/
@JsonIgnoreProperties(ignoreUnknown = true)
data class DashScopeResponse(
val output: Output?,
val usage: Usage?,
@JsonProperty("request_id")
val requestId: String?
)
/**
* 输出数据类
*
* 该类包含Dash服务生成的内容，主要是通过choices列表来提供可能的回复选项
*
* @param choices 生成内容的选项列表
*/
@JsonIgnoreProperties(ignoreUnknown = true)
data class Output(
val choices: List<Choice>?
)
/**
* 选择项数据类
*
* 该类表示生成内容的一个选项，包括完成原因和具体的消息内容
*
* @param finishReason 完成原因，解释生成内容结束的原因
* @param message 消息内容，包括角色和具体内容
*/
@JsonIgnoreProperties(ignoreUnknown = true)
data class Choice(
@JsonProperty("finish_reason")
val finishReason: String?,
val message: Message?
)
/**
* 消息数据类
*
* 该类表示一条消息，包括消息的角色（如用户、助手）和具体内容
* 具体内容通过一个内容列表来表示
*
* @param role 消息的角色，如用户、助手等
* @param content 消息的具体内容列表
*/
@JsonIgnoreProperties(ignoreUnknown = true)
data class Message(
val role: String?,
val content: List<Content>?
)
/**
* 内容数据类
*
* 该类表示消息中的具体内容，目前只支持文本内容
*
* @param text 文本内容
*/
@JsonIgnoreProperties(ignoreUnknown = true)
data class Content(
val text: String?
)
/**
* 使用情况数据类
*
* 该类详细记录了Dash服务的使用情况，包括输入和输出的令牌数量和详情
*
* @param inputTokensDetails 输入令牌的详情
* @param outputTokensDetails 输出令牌的详情
* @param inputTokens 输入令牌的数量
* @param outputTokens 输出令牌的数量
* @param totalTokens 总令牌的数量
* @param imageTokens 图像令牌的数量，如果适用
*/
@JsonIgnoreProperties(ignoreUnknown = true)
data class Usage(
@JsonProperty("input_tokens_details")
val inputTokensDetails: TokenDetail?,
@JsonProperty("output_tokens_details")
val outputTokensDetails: TokenDetail?,
@JsonProperty("input_tokens")
val inputTokens: Int?,
@JsonProperty("output_tokens")
val outputTokens: Int?,
@JsonProperty("total_tokens")
val totalTokens: Int?,
@JsonProperty("image_tokens")
val imageTokens: Int?
)
/**
* 令牌详情数据类
*
* 该类提供了令牌的详细信息，目前仅支持文本令牌的数量
*
* @param textTokens 文本令牌的数量
*/
@JsonIgnoreProperties(ignoreUnknown = true)
data class TokenDetail(
@JsonProperty("text_tokens")
val textTokens: Int?
)

11
llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/BaiLianModelService.kt
View File

@ -1,8 +1,8 @@
package org.jcnc.llmx.impl.baiLian.service
import kotlinx.coroutines.flow.Flow
import org.jcnc.llmx.core.spi.entities.request.ChatRequest
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
/**
@ -20,4 +20,11 @@ interface BaiLianModelService {
* @return 返回一个Flow流，包含部分聊天响应，允许逐步处理和消费响应
*/
fun streamChat(request: ChatRequest): Flow<ChatResponsePart>
/**
* 使用流式多模态聊天交互
*
* @param request 聊天请求对象，包含用户输入、上下文等信息
* @return 返回一个Flow流，包含部分聊天响应，允许逐步处理和消费响应
*/
fun streamMultimodalityChat(request: ChatRequest) : Flow<ChatResponsePart>
}

79
llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/impl/BaiLianModelServiceImpl.kt
View File

@ -4,11 +4,13 @@ import com.fasterxml.jackson.core.type.TypeReference
import com.fasterxml.jackson.databind.ObjectMapper
import com.gewuyou.forgeboot.core.extension.log
import kotlinx.coroutines.flow.Flow
import org.jcnc.llmx.core.spi.entities.request.ChatRequest
import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart
import org.jcnc.llmx.core.spi.entities.response.Usage
import org.jcnc.llmx.core.common.entities.request.ChatRequest
import org.jcnc.llmx.core.common.entities.response.ChatResponsePart
import org.jcnc.llmx.core.common.entities.response.Usage
import org.jcnc.llmx.impl.baiLian.adapter.DashScopeAdapter
import org.jcnc.llmx.impl.baiLian.config.entities.DashScopeProperties
import org.jcnc.llmx.impl.baiLian.entities.response.DashScopeResponse
import org.jcnc.llmx.impl.baiLian.service.BaiLianModelService
import org.springframework.stereotype.Service
@ -123,4 +125,75 @@ class BaiLianModelServiceImpl(
}
)
}
/**
* 使用流式多模态聊天交互
*
* @param request 聊天请求对象，包含用户输入、上下文等信息
* @return 返回一个Flow流，包含部分聊天响应，允许逐步处理和消费响应
*/
override fun streamMultimodalityChat(request: ChatRequest): Flow<ChatResponsePart> {
// 构造请求URL
val url = dashScopeProperties.multimodalityUrl
log.info("请求URL: $url")
// 构造请求头，包括授权信息和内容类型
val headers = mapOf(
"Authorization" to "Bearer ${dashScopeProperties.apiKey}",
"Content-Type" to "application/json",
"X-DashScope-SSE" to "enable"
)
log.info("请求头: $headers")
// 构造输入参数，主要包括用户的message
val inputMap = mutableMapOf("messages" to request.messages)
// 构造参数地图，包括是否包含思考、模型ID、增量输出等
val parametersMap = mutableMapOf(
"incremental_output" to true,
)
// 构造请求体，包括输入参数、构造的参数地图和调试信息
val body = mapOf(
"model" to request.model,
"input" to inputMap,
"parameters" to parametersMap,
)
// 发送流式聊天请求，并处理响应
return dashScopeAdapter.sendStreamChat(
url, headers, body,
{ json: String ->
val response = objectMapper.readValue(json, DashScopeResponse::class.java)
val choices = response
.output
?.choices
?.getOrNull(0)
// 提取输出文本
val text = choices
?.message
?.content
?.getOrNull(0)
?.text
?: ""
// 判断是否完成（finish_reason 通常为 "stop"，但你这里是字符串 "null"）
val finishReason = choices?.finishReason
val done = finishReason == "stop"
// 提取使用情况
val usage = response.usage
val promptTokens = usage?.inputTokens ?: 0
val completionTokens = usage?.outputTokens ?: 0
val totalTokens = usage?.totalTokens ?: 0
ChatResponsePart(
content = text,
done = done,
usage = Usage(
promptTokens = promptTokens,
completionTokens = completionTokens,
totalTokens = totalTokens
),
other = mapOf(
"request_id" to (response.requestId ?: ""),
"model" to request.model,
"response" to json
)
)
}
)
}
}

1
llmx-impl/llmx-impl-bailian/src/main/resources/application-dev.yml
View File

@ -22,3 +22,4 @@ aliyun:
api-key: sk-78af4dd964a94f4cb373851064dbdc12 # API密钥
app-id: 3fae0bbab2e54a90a37aa02cd12dd62c # 应用ID
base-url: https://dashscope.aliyuncs.com/api/v1/apps/ # 基础API URL
multimodality-url: https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation

1
llmx-impl/llmx-impl-bailian/src/main/resources/application-test.yml
View File

@ -18,3 +18,4 @@ aliyun:
api-key: sk-78af4dd964a94f4cb373851064dbdc12 # API密钥
app-id: 3fae0bbab2e54a90a37aa02cd12dd62c # 应用ID
base-url: https://dashscope.aliyuncs.com/api/v1/apps/ # 基础API URL
multimodality-url: https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation

21
llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/Base64Image.java Normal file
View File

@ -0,0 +1,21 @@
package com.gewuyou.bailian.util;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Base64;
/**
* Base 64 图像
*
* @author gewuyou
* @since 2025-05-08 18:56:02
*/
public class Base64Image {
public static void main(String[] args) throws Exception {
byte[] imageBytes = Files.readAllBytes(Paths.get("F:/gewuyou/Project/Idea/demo-llm/src/main/resources/2025俏春装/10147009251.jpg"));
String base64Image = Base64.getEncoder().encodeToString(imageBytes);
String imagePayload = "data:image/jpeg;base64," + base64Image;
System.out.println(imagePayload); // 你可以把它放到 JSON 中使用
}
}

85
llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/DashScopeLocalImageClient.java Normal file
View File

@ -0,0 +1,85 @@
package com.gewuyou.bailian.util;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.util.Base64;
/**
* DashScopeLocalImageClient
*
* @author gewuyou
* @since 2025-05-08 18:57:58
*/
public class DashScopeLocalImageClient {
public static void main(String[] args) throws Exception {
// Step 1: 读取本地图片并转成 base64
String imagePath = "F:/gewuyou/Project/Idea/demo-llm/src/main/resources/2025俏春装/10147009251.jpg";
byte[] imageBytes = Files.readAllBytes(new File(imagePath).toPath());
String base64Image = Base64.getEncoder().encodeToString(imageBytes);
String imageDataUrl = "data:image/jpeg;base64," + base64Image;
System.out.println(imageDataUrl);
// Step 2: 构造请求 JSON
String jsonBody = """
{
"model": "qwen-vl-max-latest",
"input": {
"messages": [
{
"role": "system",
"content": [
{ "text": "You are a helpful assistant." }
]
},
{
"role": "user",
"content": [
{ "image": "%s" },
{ "text": "请给这张图打标签" }
]
}
]
},
"parameters": {
"incremental_output": true
}
}
""".formatted(imageDataUrl.replace("\n", ""));
// Step 3: 发送 POST 请求
URL url = new URL("https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("Authorization", "Bearer sk-78af4dd964a94f4cb373851064dbdc12");
connection.setRequestProperty("Content-Type", "application/json");
connection.setDoOutput(true);
try (OutputStream os = connection.getOutputStream()) {
os.write(jsonBody.getBytes());
os.flush();
}
// Step 4: 读取响应
int responseCode = connection.getResponseCode();
System.out.println("Response Code: " + responseCode);
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(responseCode == 200 ?
connection.getInputStream() : connection.getErrorStream(), "GBK"))) {
String line;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
}
connection.disconnect();
}
}

14
settings.gradle.kts
View File

@ -6,15 +6,25 @@ include(
"llmx-core",
":llmx-core:llmx-core-service",
":llmx-core:llmx-core-spi",
":llmx-core:llmx-core-api",
":llmx-core:llmx-core-common",
)
project(":llmx-core:llmx-core-service").name = "llmx-core-service"
project(":llmx-core:llmx-core-spi").name = "llmx-core-spi"
project(":llmx-core:llmx-core-api").name = "llmx-core-api"
project(":llmx-core:llmx-core-common").name = "llmx-core-common"
include(
"llmx-impl",
"llmx-impl:llmx-impl-bailian",
":llmx-impl:llmx-impl-bailian",
// "llmx-impl:llmx-impl-openAi",
)
project(":llmx-impl:llmx-impl-bailian").name = "llmx-impl-bailian"
//project(":llmx-impl:llmx-impl-openAi").name = "llmx-impl-openAi"
//project(":llmx-impl:llmx-impl-openAi").name = "llmx-impl-openAi"
include(
"llmx-app",
":llmx-app:llmx-app-multimodality"
)
project(":llmx-app:llmx-app-multimodality").name = "llmx-app-multimodality"