diff --git a/.gitea/workflows/deploy.test.yml b/.gitea/workflows/deploy.test.yml index 8914aec..5b922b8 100644 --- a/.gitea/workflows/deploy.test.yml +++ b/.gitea/workflows/deploy.test.yml @@ -79,7 +79,7 @@ jobs: ./gradlew jib --stacktrace --build-cache --info -Dorg.gradle.caching=true -Dorg.gradle.jvmargs="-Xmx2g -Xms2g -XX:MaxMetaspaceSize=1g" | tee build.log echo "=== 镜像构建结果 ===" - docker images | grep ${{env.PROJECT_NAME}} + docker images | grep ${{ env.PROJECT_NAME }} || true - name: 🛑 Stop Gradle Daemon run: | echo "停止Gradle守护进程..." diff --git a/build.gradle.kts b/build.gradle.kts index 0ee7ffc..a22d08a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -23,6 +23,7 @@ allprojects { set(ProjectFlags.USE_LLM_CORE_SPI, false) set(ProjectFlags.USE_SPRING_CLOUD_BOM, false) set(ProjectFlags.IS_ROOT_MODULE, false) + set(ProjectFlags.USE_SPRING_BOOT_BOM,false) } repositories { mavenLocal() @@ -93,6 +94,11 @@ subprojects { implementation(platform(libs.springCloudDependencies.bom)) } } + if(project.getPropertyByBoolean(ProjectFlags.USE_SPRING_BOOT_BOM)){ + dependencies { + implementation(platform(libs.springBootDependencies.bom)) + } + } } val libs = rootProject.libs apply { diff --git a/buildSrc/src/main/kotlin/Modules.kt b/buildSrc/src/main/kotlin/Modules.kt index dacb471..6b0d211 100644 --- a/buildSrc/src/main/kotlin/Modules.kt +++ b/buildSrc/src/main/kotlin/Modules.kt @@ -14,6 +14,8 @@ object Modules { object Core{ // llmx-core-spi模块的路径，用于定义核心功能的SPI const val SPI = ":llmx-core:llmx-core-spi" + const val API = ":llmx-core:llmx-core-api" + const val COMMON = ":llmx-core:llmx-core-common" } } diff --git a/buildSrc/src/main/kotlin/ProjectFlags.kt b/buildSrc/src/main/kotlin/ProjectFlags.kt index db93d4a..c00120c 100644 --- a/buildSrc/src/main/kotlin/ProjectFlags.kt +++ b/buildSrc/src/main/kotlin/ProjectFlags.kt @@ -1,5 +1,6 @@ object ProjectFlags { const val USE_SPRING_BOOT_WEB = "useSpringBootWeb" + const val USE_SPRING_BOOT_BOM = "useSpringBootBom" const val USE_SPRING_CLOUD_BOM = "useSpringCloudBom" const val USE_LLM_CORE_SPI = "useLLMCoreSPI" const val IS_ROOT_MODULE = "isRootModule" diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b965abb..d6c2bde 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,6 +8,7 @@ spring-cloud-starter-alibaba-nacos-discovery-version = "2023.0.1.0" forgeBoot-version = "1.1.0-SNAPSHOT" okHttp-version = "4.12.0" jib-version = "3.4.2" +org-reactivestreams-reactiveStreams-version = "1.0.4" [plugins] # 应用 Java 插件，提供基本的 Java 代码编译和构建能力 java = { id = "java" } @@ -31,6 +32,7 @@ jibLocalPlugin = { id = "org.jcnc.llmx.plugin.jib" } jib-gradlePlugin = { module = "com.google.cloud.tools.jib:com.google.cloud.tools.jib.gradle.plugin", version.ref = "jib-version" } # bom springCloudDependencies-bom = { group ="org.springframework.cloud",name = "spring-cloud-dependencies", version.ref = "spring-cloud-version" } +springBootDependencies-bom = { module = "org.springframework.boot:spring-boot-dependencies", version.ref = "spring-boot-version" } # kotlinx # 响应式协程库 kotlinx-coruntes-reactor = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-reactor" } @@ -41,6 +43,7 @@ aliyun-bailian = { group = "com.aliyun", name = "bailian20231229", version.ref = # SrpingCloud springCloudStarter-alibaba-nacos-discovery = { group = "com.alibaba.cloud", name = "spring-cloud-starter-alibaba-nacos-discovery", version.ref = "spring-cloud-starter-alibaba-nacos-discovery-version" } springCloudStarter-loadbalancer = { group = "org.springframework.cloud", name = "spring-cloud-starter-loadbalancer" } +springCloudStarter-openfeign = { group = "org.springframework.cloud", name = "spring-cloud-starter-openfeign" } # SpringBootStarter springBootStarter-webflux = { group = "org.springframework.boot", name = "spring-boot-starter-webflux" } @@ -49,6 +52,8 @@ springBootStarter-test = { group = "org.springframework.boot", name = "spring-bo junitPlatform-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" } +# org-reactivestreams +org-reactivestreams-reactiveStreams = { group = "org.reactivestreams", name = "reactive-streams" ,version.ref="org-reactivestreams-reactiveStreams-version"} # OkHttp okHttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okHttp-version" } # forgeBoot diff --git a/gradlew b/gradlew index f3b75f3..e93a03e 100644 --- a/gradlew +++ b/gradlew @@ -207,7 +207,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: # * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. -# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# * For gewuyou: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. set -- \ diff --git a/llmx-app/.gitattributes b/llmx-app/.gitattributes new file mode 100644 index 0000000..8af972c --- /dev/null +++ b/llmx-app/.gitattributes @@ -0,0 +1,3 @@ +/gradlew text eol=lf +*.bat text eol=crlf +*.jar binary diff --git a/llmx-app/.gitignore b/llmx-app/.gitignore new file mode 100644 index 0000000..5a979af --- /dev/null +++ b/llmx-app/.gitignore @@ -0,0 +1,40 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Kotlin ### +.kotlin diff --git a/llmx-app/build.gradle.kts b/llmx-app/build.gradle.kts new file mode 100644 index 0000000..d1a1567 --- /dev/null +++ b/llmx-app/build.gradle.kts @@ -0,0 +1,9 @@ +dependencies { + +} +/** + * 由于 Kotlin 插件引入时会自动添加依赖，但根项目不需要 Kotlin 依赖，因此需要排除 Kotlin 依赖 + */ +configurations.implementation { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib") +} diff --git a/llmx-app/llmx-app-multimodality/.gitattributes b/llmx-app/llmx-app-multimodality/.gitattributes new file mode 100644 index 0000000..8af972c --- /dev/null +++ b/llmx-app/llmx-app-multimodality/.gitattributes @@ -0,0 +1,3 @@ +/gradlew text eol=lf +*.bat text eol=crlf +*.jar binary diff --git a/llmx-app/llmx-app-multimodality/.gitignore b/llmx-app/llmx-app-multimodality/.gitignore new file mode 100644 index 0000000..5a979af --- /dev/null +++ b/llmx-app/llmx-app-multimodality/.gitignore @@ -0,0 +1,40 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Kotlin ### +.kotlin diff --git a/llmx-app/llmx-app-multimodality/build.gradle.kts b/llmx-app/llmx-app-multimodality/build.gradle.kts new file mode 100644 index 0000000..af933bf --- /dev/null +++ b/llmx-app/llmx-app-multimodality/build.gradle.kts @@ -0,0 +1,13 @@ +extra { + // 开启springboot + setProperty(ProjectFlags.USE_SPRING_BOOT_WEB, true) + setProperty(ProjectFlags.USE_SPRING_CLOUD_BOM, true) +} +dependencies { + // Nacos 服务发现和配置 + implementation(libs.springCloudStarter.alibaba.nacos.discovery) + implementation(libs.forgeBoot.core.extension) + implementation(libs.forgeBoot.webmvc.version.springBootStarter) + implementation(libs.jackson.module.kotlin) + implementation(project(Modules.Core.API)) +} diff --git a/llmx-app/llmx-app-multimodality/src/main/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplication.kt b/llmx-app/llmx-app-multimodality/src/main/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplication.kt new file mode 100644 index 0000000..8133bab --- /dev/null +++ b/llmx-app/llmx-app-multimodality/src/main/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplication.kt @@ -0,0 +1,11 @@ +package org.jcnc.llmx.app.multimodality + +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.runApplication + +@SpringBootApplication +class LlmxAppMultimodalityApplication + +fun main(args: Array) { + runApplication(*args) +} diff --git a/llmx-app/llmx-app-multimodality/src/main/resources/application-dev.yml b/llmx-app/llmx-app-multimodality/src/main/resources/application-dev.yml new file mode 100644 index 0000000..3360232 --- /dev/null +++ b/llmx-app/llmx-app-multimodality/src/main/resources/application-dev.yml @@ -0,0 +1,8 @@ +server: + port: 8081 +spring: + config: + import: classpath:bootstrap-dev.yml + + + diff --git a/llmx-core/llmx-core-service/src/main/resources/application-prod.yml b/llmx-app/llmx-app-multimodality/src/main/resources/application-master.yml similarity index 100% rename from llmx-core/llmx-core-service/src/main/resources/application-prod.yml rename to llmx-app/llmx-app-multimodality/src/main/resources/application-master.yml diff --git a/llmx-app/llmx-app-multimodality/src/main/resources/application-test.yml b/llmx-app/llmx-app-multimodality/src/main/resources/application-test.yml new file mode 100644 index 0000000..d75e277 --- /dev/null +++ b/llmx-app/llmx-app-multimodality/src/main/resources/application-test.yml @@ -0,0 +1,7 @@ +server: + port: 9003 +spring: + config: + import: classpath:bootstrap-test.yml + + diff --git a/llmx-app/llmx-app-multimodality/src/main/resources/application.yml b/llmx-app/llmx-app-multimodality/src/main/resources/application.yml new file mode 100644 index 0000000..9c9db5f --- /dev/null +++ b/llmx-app/llmx-app-multimodality/src/main/resources/application.yml @@ -0,0 +1,5 @@ +spring: + application: + name: llmx-app-multimodality + profiles: + active: dev diff --git a/llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-dev.yml b/llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-dev.yml new file mode 100644 index 0000000..37654f9 --- /dev/null +++ b/llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-dev.yml @@ -0,0 +1,12 @@ +spring: + cloud: + nacos: + username: nacos + password: L4s6f9y3 + server-addr: 49.235.96.75:8848 + ip: 192.168.1.100 + discovery: + server-addr: ${spring.cloud.nacos.server-addr} + username: ${spring.cloud.nacos.username} + password: ${spring.cloud.nacos.password} + ip: ${spring.cloud.nacos.ip} \ No newline at end of file diff --git a/llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-test.yml b/llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-test.yml new file mode 100644 index 0000000..c9dc5c7 --- /dev/null +++ b/llmx-app/llmx-app-multimodality/src/main/resources/bootstrap-test.yml @@ -0,0 +1,10 @@ +spring: + cloud: + nacos: + username: nacos + password: L4s6f9y3 + server-addr: 49.235.96.75:9001 + discovery: + server-addr: ${spring.cloud.nacos.server-addr} + username: ${spring.cloud.nacos.username} + password: ${spring.cloud.nacos.password} \ No newline at end of file diff --git a/llmx-app/llmx-app-multimodality/src/test/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplicationTests.kt b/llmx-app/llmx-app-multimodality/src/test/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplicationTests.kt new file mode 100644 index 0000000..98e9d18 --- /dev/null +++ b/llmx-app/llmx-app-multimodality/src/test/kotlin/org/jcnc/llmx/app/multimodality/LlmxAppMultimodalityApplicationTests.kt @@ -0,0 +1,13 @@ +package org.jcnc.llmx.app.multimodality + +import org.junit.jupiter.api.Test +import org.springframework.boot.test.context.SpringBootTest + +@SpringBootTest +class LlmxAppMultimodalityApplicationTests { + + @Test + fun contextLoads() { + } + +} diff --git a/llmx-core/build.gradle.kts b/llmx-core/build.gradle.kts index e972465..d1a1567 100644 --- a/llmx-core/build.gradle.kts +++ b/llmx-core/build.gradle.kts @@ -2,7 +2,7 @@ dependencies { } /** - * 由于 Kotlin 插件被引入时会自动添加依赖，但根项目不需要 Kotlin 依赖，因此需要排除 Kotlin 依赖 + * 由于 Kotlin 插件引入时会自动添加依赖，但根项目不需要 Kotlin 依赖，因此需要排除 Kotlin 依赖 */ configurations.implementation { exclude(group = "org.jetbrains.kotlin", module = "kotlin-stdlib") diff --git a/llmx-core/llmx-core-api/.gitattributes b/llmx-core/llmx-core-api/.gitattributes new file mode 100644 index 0000000..8af972c --- /dev/null +++ b/llmx-core/llmx-core-api/.gitattributes @@ -0,0 +1,3 @@ +/gradlew text eol=lf +*.bat text eol=crlf +*.jar binary diff --git a/llmx-core/llmx-core-api/.gitignore b/llmx-core/llmx-core-api/.gitignore new file mode 100644 index 0000000..5a979af --- /dev/null +++ b/llmx-core/llmx-core-api/.gitignore @@ -0,0 +1,40 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Kotlin ### +.kotlin diff --git a/llmx-core/llmx-core-api/build.gradle.kts b/llmx-core/llmx-core-api/build.gradle.kts new file mode 100644 index 0000000..ccff2b8 --- /dev/null +++ b/llmx-core/llmx-core-api/build.gradle.kts @@ -0,0 +1,10 @@ +ext{ + set(ProjectFlags.USE_SPRING_CLOUD_BOM,true) +} +dependencies { + api(project(Modules.Core.COMMON)) + api(libs.org.reactivestreams.reactiveStreams) + // Spring Cloud OpenFeign + implementation(libs.springCloudStarter.openfeign) +} + diff --git a/llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignAutoConfiguration.kt b/llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignAutoConfiguration.kt new file mode 100644 index 0000000..54052b2 --- /dev/null +++ b/llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignAutoConfiguration.kt @@ -0,0 +1,14 @@ +package org.jcnc.llmx.core.api + +import org.springframework.cloud.openfeign.EnableFeignClients +import org.springframework.context.annotation.Configuration + +/** + *LLM 核心Feign自动配置 + * + * @since 2025-05-08 16:48:23 + * @author gewuyou + */ +@Configuration +@EnableFeignClients(basePackages = ["org.jcnc.llmx.core.api"]) +open class LLMCoreFeignAutoConfiguration \ No newline at end of file diff --git a/llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignClient.kt b/llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignClient.kt new file mode 100644 index 0000000..f1257b3 --- /dev/null +++ b/llmx-core/llmx-core-api/src/main/kotlin/org/jcnc/llmx/core/api/LLMCoreFeignClient.kt @@ -0,0 +1,21 @@ +package org.jcnc.llmx.core.api + +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart +import org.reactivestreams.Publisher +import org.springframework.cloud.openfeign.FeignClient +import org.springframework.http.MediaType +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody + +/** + *LLM 核心客户端 + * + * @since 2025-05-08 16:22:22 + * @author gewuyou + */ +@FeignClient(name = "llmx-core-service") // name 对应 Nacos 中注册的服务名 +fun interface LLMCoreFeignClient { + @PostMapping("/api/v1/chat/stream", consumes = [MediaType.APPLICATION_JSON_VALUE]) + fun chat(@RequestBody request: ChatRequest): Publisher +} \ No newline at end of file diff --git a/llmx-core/llmx-core-api/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/llmx-core/llmx-core-api/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000..4b731f2 --- /dev/null +++ b/llmx-core/llmx-core-api/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +org.jcnc.llmx.core.api.LLMCoreFeignAutoConfiguration \ No newline at end of file diff --git a/llmx-core/llmx-core-common/.gitattributes b/llmx-core/llmx-core-common/.gitattributes new file mode 100644 index 0000000..8af972c --- /dev/null +++ b/llmx-core/llmx-core-common/.gitattributes @@ -0,0 +1,3 @@ +/gradlew text eol=lf +*.bat text eol=crlf +*.jar binary diff --git a/llmx-core/llmx-core-common/.gitignore b/llmx-core/llmx-core-common/.gitignore new file mode 100644 index 0000000..5a979af --- /dev/null +++ b/llmx-core/llmx-core-common/.gitignore @@ -0,0 +1,40 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Kotlin ### +.kotlin diff --git a/llmx-core/llmx-core-common/build.gradle.kts b/llmx-core/llmx-core-common/build.gradle.kts new file mode 100644 index 0000000..f9b855f --- /dev/null +++ b/llmx-core/llmx-core-common/build.gradle.kts @@ -0,0 +1,7 @@ +ext{ + set(ProjectFlags.USE_SPRING_BOOT_BOM,true) +} +dependencies { + implementation(libs.jackson.core) + implementation(libs.jackson.module.kotlin) +} diff --git a/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/deserializer/MultiModalContentDeserializer.kt b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/deserializer/MultiModalContentDeserializer.kt new file mode 100644 index 0000000..d12674f --- /dev/null +++ b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/deserializer/MultiModalContentDeserializer.kt @@ -0,0 +1,42 @@ +package org.jcnc.llmx.core.common.deserializer + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jcnc.llmx.core.common.entities.request.MultiModalContent + +/** + * 多模态内容反序列化器 + * + * 该类用于将JSON表示的多模态内容反序列化为MultiModalContent数据结构 + * 主要处理两种情况：文本内容和图像内容，并相应地创建Text或Image实例 + * 如果JSON中不包含已知的内容类型，则抛出IllegalArgumentException + * + * @since 2025-05-08 20:41:15 + * @author gewuyou + */ +class MultiModalContentDeserializer : JsonDeserializer() { + /** + * 反序列化方法 + * + * 根据JSON节点中的内容类型，创建相应的MultiModalContent实例 + * 如果内容类型未知，则抛出异常 + * + * @param p JsonParser对象，用于解析JSON输入 + * @param ctxt DeserializationContext对象，提供反序列化上下文 + * @return MultiModalContent实例，表示反序列化的多模态内容 + * @throws IllegalArgumentException 如果JSON节点中的内容类型未知 + */ + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): MultiModalContent { + // 解析JSON输入并将其读取为ObjectNode + val node: ObjectNode = p.codec.readTree(p) + + // 根据JSON节点中的内容类型，返回相应的MultiModalContent实例 + return when { + node.has("text") -> MultiModalContent.Text(node["text"].asText()) + node.has("image") -> MultiModalContent.Image(node["image"].asText()) + else -> throw IllegalArgumentException("Unknown content type in: $node") + } + } +} diff --git a/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/ChatRequest.kt b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/ChatRequest.kt new file mode 100644 index 0000000..28be69d --- /dev/null +++ b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/ChatRequest.kt @@ -0,0 +1,52 @@ +package org.jcnc.llmx.core.common.entities.request + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import org.jcnc.llmx.core.common.deserializer.MultiModalContentDeserializer + +/** + * MultiModalContent密封类，用于定义消息内容的多种模态 + * 这是一个密封类，意味着所有的子类都需要在这个文件中定义 + */ +@JsonDeserialize(using = MultiModalContentDeserializer::class) +sealed class MultiModalContent { + /** + * Text数据类，表示文本内容的模态 + * @param text 具体的文本内容 + */ + data class Text(val text: String) : MultiModalContent() + + /** + * Image数据类，表示图像内容的模态 + * @param image 图像的base64 + */ + data class Image(val image: String) : MultiModalContent() +} + +/** + * ChatRequest数据类，用于封装聊天请求的参数 + * @since 2025-04-25 17:07:02 + * @author gewuyou + * @param prompt 用户的聊天提示或消息，是聊天请求的主要输入 + * @param model 使用的聊天模型名称，决定了解析和响应的方式 + * @param messages 包含多个模态消息的列表，每个消息有特定的角色和内容 + * @param options 可选的额外参数集合，用于定制聊天请求的行为和输出 + * 可以包括如最大回复长度、温度（随机性）等 + */ +data class ChatRequest( + val prompt: String? = "", + val model: String, + val messages: List =listOf(), + val options: Map = mapOf() +) + +/** + * MultiModalMessage数据类，用于定义聊天请求中的单个消息 + * @param role 消息的角色，可以是"system", "user", "assistant"等 + * @param content 消息的内容，可以是文本、图像等多种模态的组合 + */ +data class MultiModalMessage( + val role: String, // "system", "user", "assistant" + val content: List +) + + diff --git a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/EmbeddingRequest.kt b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/EmbeddingRequest.kt similarity index 90% rename from llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/EmbeddingRequest.kt rename to llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/EmbeddingRequest.kt index d30ea3d..6a9603f 100644 --- a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/EmbeddingRequest.kt +++ b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/request/EmbeddingRequest.kt @@ -1,4 +1,4 @@ -package org.jcnc.llmx.core.spi.entities.request +package org.jcnc.llmx.core.common.entities.request /** * 嵌入请求类 diff --git a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/ChatResponse.kt b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/ChatResponse.kt similarity index 95% rename from llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/ChatResponse.kt rename to llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/ChatResponse.kt index f0ca30c..4fb9730 100644 --- a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/ChatResponse.kt +++ b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/ChatResponse.kt @@ -1,4 +1,4 @@ -package org.jcnc.llmx.core.spi.entities.response +package org.jcnc.llmx.core.common.entities.response /** * 聊天响应类 diff --git a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/EmbeddingResponse.kt b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/EmbeddingResponse.kt similarity index 91% rename from llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/EmbeddingResponse.kt rename to llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/EmbeddingResponse.kt index fa60996..09ee7c8 100644 --- a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/response/EmbeddingResponse.kt +++ b/llmx-core/llmx-core-common/src/main/kotlin/org/jcnc/llmx/core/common/entities/response/EmbeddingResponse.kt @@ -1,4 +1,4 @@ -package org.jcnc.llmx.core.spi.entities.response +package org.jcnc.llmx.core.common.entities.response /** * 嵌入响应类 diff --git a/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/controller/ChatController.kt b/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/controller/ChatController.kt index bf3570a..e905147 100644 --- a/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/controller/ChatController.kt +++ b/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/controller/ChatController.kt @@ -1,10 +1,12 @@ package org.jcnc.llmx.core.service.controller +import com.gewuyou.forgeboot.core.extension.log import com.gewuyou.forgeboot.webmvc.version.annotation.ApiVersion -import kotlinx.coroutines.flow.Flow +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart import org.jcnc.llmx.core.service.service.impl.LLMServiceImpl -import org.jcnc.llmx.core.spi.entities.request.ChatRequest -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart + +import org.reactivestreams.Publisher import org.springframework.http.MediaType import org.springframework.web.bind.annotation.PostMapping @@ -35,7 +37,22 @@ class ChatController( * @return 返回一个Flow流，流中依次提供了聊天响应的部分数据，允许异步处理和逐步消费响应内容 */ @PostMapping("/stream", produces = [MediaType.APPLICATION_NDJSON_VALUE]) - fun chat(@RequestBody request: ChatRequest): Flow { + fun chat(@RequestBody request: ChatRequest): Publisher { return llmServiceImpl.chat(request) } + /** + * 处理多模态聊天请求的函数 + * 该函数通过 POST 方法接收一个聊天请求，并以 NDJSON 的形式返回聊天响应的部分 + * + * @param request 包含聊天请求信息的数据类 + * @return 返回一个发布者，用于异步地发送聊天响应的部分 + * + * 注意：该函数被设计为异步处理，以提高性能和响应性 + * 它使用了响应式编程模型，适合处理高并发和大数据量的响应 + */ + @PostMapping("/streamMultimodality", produces = [MediaType.APPLICATION_NDJSON_VALUE]) + fun multimodalityChat(@RequestBody request: ChatRequest): Publisher{ + log.info("request: {}", request) + return llmServiceImpl.multimodalityChat(request) + } } \ No newline at end of file diff --git a/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/LLMService.kt b/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/LLMService.kt index badf32f..bd0ae6f 100644 --- a/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/LLMService.kt +++ b/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/LLMService.kt @@ -1,8 +1,9 @@ package org.jcnc.llmx.core.service.service -import kotlinx.coroutines.flow.Flow -import org.jcnc.llmx.core.spi.entities.request.ChatRequest -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart + +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart +import org.reactivestreams.Publisher import org.springframework.web.bind.annotation.PostMapping @@ -12,7 +13,7 @@ import org.springframework.web.bind.annotation.PostMapping * @since 2025-04-26 17:38:18 * @author gewuyou */ -fun interface LLMService { +interface LLMService { /** * 初始化与聊天服务的连接，以处理聊天请求 * @@ -23,5 +24,6 @@ fun interface LLMService { * @return 返回一个Flow流，通过该流可以接收到聊天响应的部分数据，如消息、状态更新等 */ @PostMapping("/chat") - fun chat(request: ChatRequest): Flow + fun chat(request: ChatRequest): Publisher + fun multimodalityChat(request: ChatRequest): Publisher } \ No newline at end of file diff --git a/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/impl/LLMServiceImpl.kt b/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/impl/LLMServiceImpl.kt index cdf45d8..2222156 100644 --- a/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/impl/LLMServiceImpl.kt +++ b/llmx-core/llmx-core-service/src/main/kotlin/org/jcnc/llmx/core/service/service/impl/LLMServiceImpl.kt @@ -1,11 +1,11 @@ package org.jcnc.llmx.core.service.service.impl -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.reactive.asFlow + +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart import org.jcnc.llmx.core.service.manager.ModelRouteManager import org.jcnc.llmx.core.service.service.LLMService -import org.jcnc.llmx.core.spi.entities.request.ChatRequest -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart +import org.reactivestreams.Publisher import org.springframework.stereotype.Service import org.springframework.web.reactive.function.client.WebClient @@ -30,7 +30,7 @@ class LLMServiceImpl( * @param request 聊天请求对象，包含建立聊天所需的信息，如用户标识、会话标识等 * @return 返回一个Flow流，通过该流可以接收到聊天响应的部分数据，如消息、状态更新等 */ - override fun chat(request: ChatRequest): Flow { + override fun chat(request: ChatRequest): Publisher { val serviceName = modelRouteManager.resolveServiceName(request.model) val webClient = webClientBuilder.build() return webClient.post() @@ -38,6 +38,15 @@ class LLMServiceImpl( .bodyValue(request) .retrieve() .bodyToFlux(ChatResponsePart::class.java) - .asFlow() + } + + override fun multimodalityChat(request: ChatRequest): Publisher { + val serviceName = modelRouteManager.resolveServiceName(request.model) + val webClient = webClientBuilder.build() + return webClient.post() + .uri("http://$serviceName/provider/multimodalityChat") + .bodyValue(request) + .retrieve() + .bodyToFlux(ChatResponsePart::class.java) } } \ No newline at end of file diff --git a/llmx-core/llmx-core-service/src/main/resources/application-dev.yml b/llmx-core/llmx-core-service/src/main/resources/application-dev.yml index 4eb3cbd..d5844fb 100644 --- a/llmx-core/llmx-core-service/src/main/resources/application-dev.yml +++ b/llmx-core/llmx-core-service/src/main/resources/application-dev.yml @@ -9,5 +9,6 @@ llmx: qwen-turbo: llmx-impl-baiLian qwen-max: llmx-impl-baiLian qwen-plus: llmx-impl-baiLian + qwen-vl-max-latest: llmx-impl-baiLian diff --git a/llmx-core/llmx-core-service/src/main/resources/application-master.yml b/llmx-core/llmx-core-service/src/main/resources/application-master.yml new file mode 100644 index 0000000..e78b1c6 --- /dev/null +++ b/llmx-core/llmx-core-service/src/main/resources/application-master.yml @@ -0,0 +1,7 @@ +server: + port: 9002 +spring: + cloud: + nacos: + discovery: + server-addr: 49.235.96.75:9001 \ No newline at end of file diff --git a/llmx-core/llmx-core-service/src/main/resources/application-test.yml b/llmx-core/llmx-core-service/src/main/resources/application-test.yml index 0eca13f..e9166e1 100644 --- a/llmx-core/llmx-core-service/src/main/resources/application-test.yml +++ b/llmx-core/llmx-core-service/src/main/resources/application-test.yml @@ -9,5 +9,6 @@ llmx: qwen-turbo: llmx-impl-baiLian qwen-max: llmx-impl-baiLian qwen-plus: llmx-impl-baiLian + qwen-vl-max-latest: llmx-impl-baiLian diff --git a/llmx-core/llmx-core-spi/build.gradle.kts b/llmx-core/llmx-core-spi/build.gradle.kts index 923d5d7..8821f83 100644 --- a/llmx-core/llmx-core-spi/build.gradle.kts +++ b/llmx-core/llmx-core-spi/build.gradle.kts @@ -4,6 +4,7 @@ apply { plugin(libs.plugins.kotlin.plugin.spring.get().pluginId) } dependencies { - compileOnly(libs.kotlinx.coruntes.reactor) compileOnly(libs.springBootStarter.web) + api(libs.org.reactivestreams.reactiveStreams) + api(project(Modules.Core.COMMON)) } \ No newline at end of file diff --git a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/ChatRequest.kt b/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/ChatRequest.kt deleted file mode 100644 index ada4e27..0000000 --- a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/entities/request/ChatRequest.kt +++ /dev/null @@ -1,17 +0,0 @@ -package org.jcnc.llmx.core.spi.entities.request - - -/** - * ChatRequest数据类，用于封装聊天请求的参数 - * @since 2025-04-25 17:07:02 - * @author gewuyou - * @param prompt 用户的聊天提示或消息，是聊天请求的主要输入 - * @param model 使用的聊天模型名称，决定了解析和响应的方式 - * @param options 可选的额外参数集合，用于定制聊天请求的行为和输出 - * 可以包括如最大回复长度、温度（随机性）等 - */ -data class ChatRequest( - val prompt: String, - val model: String, - val options: Map = mapOf() -) diff --git a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/provider/LLMProvider.kt b/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/provider/LLMProvider.kt index 404c127..69c41fb 100644 --- a/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/provider/LLMProvider.kt +++ b/llmx-core/llmx-core-spi/src/main/kotlin/org/jcnc/llmx/core/spi/provider/LLMProvider.kt @@ -1,12 +1,12 @@ package org.jcnc.llmx.core.spi.provider -import kotlinx.coroutines.flow.Flow -import org.jcnc.llmx.core.spi.entities.request.ChatRequest -import org.jcnc.llmx.core.spi.entities.request.EmbeddingRequest -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart -import org.jcnc.llmx.core.spi.entities.response.EmbeddingResponse -import org.springframework.http.MediaType +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.request.EmbeddingRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart +import org.jcnc.llmx.core.common.entities.response.EmbeddingResponse +import org.reactivestreams.Publisher +import org.springframework.http.MediaType import org.springframework.web.bind.annotation.PostMapping import org.springframework.web.bind.annotation.RequestMapping @@ -31,8 +31,19 @@ interface LLMProvider { * @return 返回一个Flow流，通过该流可以接收到聊天响应的部分数据，如消息、状态更新等 */ @PostMapping("/chat", produces = [MediaType.APPLICATION_NDJSON_VALUE]) - fun chat(request: ChatRequest): Flow - + fun chat(request: ChatRequest): Publisher + /** + * 处理多模态聊天请求的函数 + * 该函数通过 POST 方法接收一个聊天请求，并以 NDJSON 的形式返回聊天响应的部分 + * + * @param request 包含聊天请求信息的数据类 + * @return 返回一个发布者，用于异步地发送聊天响应的部分 + * + * 注意：该函数被设计为异步处理，以提高性能和响应性 + * 它使用了响应式编程模型，适合处理高并发和大数据量的响应 + */ + @PostMapping("/multimodalityChat", produces = [MediaType.APPLICATION_NDJSON_VALUE]) + fun multimodalityChat(request: ChatRequest): Publisher /** * 嵌入功能方法 * 该方法允许用户发送嵌入请求，以获取LLM生成的嵌入向量 diff --git a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/adapter/DashScopeAdapter.kt b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/adapter/DashScopeAdapter.kt index 64e229a..1d83300 100644 --- a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/adapter/DashScopeAdapter.kt +++ b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/adapter/DashScopeAdapter.kt @@ -11,7 +11,8 @@ import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart + import org.springframework.stereotype.Component import java.io.BufferedReader @@ -30,7 +31,7 @@ import java.io.BufferedReader @Component class DashScopeAdapter( private val okHttpClient: OkHttpClient, - private val objectMapper: ObjectMapper + private val objectMapper: ObjectMapper, ) { /** * 发送流式聊天请求 @@ -50,7 +51,7 @@ class DashScopeAdapter( headers: Map, requestBody: Any, extractContent: (String) -> ChatResponsePart, - dispatcher: CoroutineDispatcher = Dispatchers.IO + dispatcher: CoroutineDispatcher = Dispatchers.IO, ): Flow = flow { val requestJson = objectMapper.writeValueAsString(requestBody) log.info("📤 请求参数: {}", requestJson) @@ -59,6 +60,7 @@ class DashScopeAdapter( okHttpClient.newCall(request).execute().use { response -> if (!response.isSuccessful) { + log.error("🚨 DashScope 请求失败: code ${response.code} message: ${response.message} body: ${response.body}") throw RuntimeException("❌ DashScope 请求失败: HTTP ${response.code}") } @@ -92,7 +94,7 @@ class DashScopeAdapter( dispatcher: CoroutineDispatcher, reader: BufferedReader, extractContent: (String) -> ChatResponsePart, - allContent: StringBuilder + allContent: StringBuilder, ) { while (currentCoroutineContext().isActive) { val line = withContext(dispatcher) { diff --git a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/config/entities/DashScopeProperties.kt b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/config/entities/DashScopeProperties.kt index 608d3b5..7b79431 100644 --- a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/config/entities/DashScopeProperties.kt +++ b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/config/entities/DashScopeProperties.kt @@ -44,5 +44,9 @@ class DashScopeProperties { * 应用调用基路径，例如 /api/v1。 */ var baseUrl: String = "" + /** + * 多模态对话调用路径 + */ + var multimodalityUrl: String = "" } diff --git a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/controller/BaiLianProvider.kt b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/controller/BaiLianProvider.kt index 028c275..039502e 100644 --- a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/controller/BaiLianProvider.kt +++ b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/controller/BaiLianProvider.kt @@ -1,16 +1,20 @@ package org.jcnc.llmx.impl.baiLian.controller -import kotlinx.coroutines.flow.Flow -import org.jcnc.llmx.core.spi.entities.request.ChatRequest -import org.jcnc.llmx.core.spi.entities.request.EmbeddingRequest -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart -import org.jcnc.llmx.core.spi.entities.response.EmbeddingResponse + +import kotlinx.coroutines.reactive.asPublisher +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.request.EmbeddingRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart +import org.jcnc.llmx.core.common.entities.response.EmbeddingResponse + import org.jcnc.llmx.core.spi.provider.LLMProvider import org.jcnc.llmx.impl.baiLian.service.BaiLianModelService +import org.reactivestreams.Publisher import org.springframework.web.bind.annotation.RequestBody import org.springframework.web.bind.annotation.RestController + /** *百炼提供商 * @@ -18,10 +22,9 @@ import org.springframework.web.bind.annotation.RestController * @author gewuyou */ @RestController -//@RequestMapping("/provider") class BaiLianProvider( private val baiLianModelService: BaiLianModelService -): LLMProvider { +) : LLMProvider { /** * 初始化与聊天服务的连接，以处理聊天请求 * @@ -31,9 +34,12 @@ class BaiLianProvider( * @param request 聊天请求对象，包含建立聊天所需的信息，如用户标识、会话标识等 * @return 返回一个Flow流，通过该流可以接收到聊天响应的部分数据，如消息、状态更新等 */ -// @PostMapping("/chat") - override fun chat(@RequestBody request: ChatRequest): Flow { - return baiLianModelService.streamChat(request) + override fun chat(@RequestBody request: ChatRequest): Publisher { + return baiLianModelService.streamChat(request).asPublisher() + } + + override fun multimodalityChat(@RequestBody request: ChatRequest): Publisher { + return baiLianModelService.streamMultimodalityChat(request).asPublisher() } /** diff --git a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/entities/response/DashScopeResponse.kt b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/entities/response/DashScopeResponse.kt new file mode 100644 index 0000000..8c7c907 --- /dev/null +++ b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/entities/response/DashScopeResponse.kt @@ -0,0 +1,117 @@ +package org.jcnc.llmx.impl.baiLian.entities.response + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * Dash范围响应类 + * + * 该类用于表示从Dash服务接收到的响应数据结构它包含了响应的输出信息、使用情况和请求ID + * 主要用于JSON序列化和反序列化 + * + * @param output 输出信息，包括生成的内容和完成原因 + * @param usage 使用情况，包括输入和输出的令牌详情 + * @param requestId 请求ID，用于跟踪和调试请求 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class DashScopeResponse( + val output: Output?, + val usage: Usage?, + @JsonProperty("request_id") + val requestId: String? +) + +/** + * 输出数据类 + * + * 该类包含Dash服务生成的内容，主要是通过choices列表来提供可能的回复选项 + * + * @param choices 生成内容的选项列表 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class Output( + val choices: List? +) + +/** + * 选择项数据类 + * + * 该类表示生成内容的一个选项，包括完成原因和具体的消息内容 + * + * @param finishReason 完成原因，解释生成内容结束的原因 + * @param message 消息内容，包括角色和具体内容 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class Choice( + @JsonProperty("finish_reason") + val finishReason: String?, + val message: Message? +) + +/** + * 消息数据类 + * + * 该类表示一条消息，包括消息的角色（如用户、助手）和具体内容 + * 具体内容通过一个内容列表来表示 + * + * @param role 消息的角色，如用户、助手等 + * @param content 消息的具体内容列表 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class Message( + val role: String?, + val content: List? +) + +/** + * 内容数据类 + * + * 该类表示消息中的具体内容，目前只支持文本内容 + * + * @param text 文本内容 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class Content( + val text: String? +) + +/** + * 使用情况数据类 + * + * 该类详细记录了Dash服务的使用情况，包括输入和输出的令牌数量和详情 + * + * @param inputTokensDetails 输入令牌的详情 + * @param outputTokensDetails 输出令牌的详情 + * @param inputTokens 输入令牌的数量 + * @param outputTokens 输出令牌的数量 + * @param totalTokens 总令牌的数量 + * @param imageTokens 图像令牌的数量，如果适用 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class Usage( + @JsonProperty("input_tokens_details") + val inputTokensDetails: TokenDetail?, + @JsonProperty("output_tokens_details") + val outputTokensDetails: TokenDetail?, + @JsonProperty("input_tokens") + val inputTokens: Int?, + @JsonProperty("output_tokens") + val outputTokens: Int?, + @JsonProperty("total_tokens") + val totalTokens: Int?, + @JsonProperty("image_tokens") + val imageTokens: Int? +) + +/** + * 令牌详情数据类 + * + * 该类提供了令牌的详细信息，目前仅支持文本令牌的数量 + * + * @param textTokens 文本令牌的数量 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class TokenDetail( + @JsonProperty("text_tokens") + val textTokens: Int? +) diff --git a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/BaiLianModelService.kt b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/BaiLianModelService.kt index 3f6a488..d6c6a6b 100644 --- a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/BaiLianModelService.kt +++ b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/BaiLianModelService.kt @@ -1,8 +1,8 @@ package org.jcnc.llmx.impl.baiLian.service import kotlinx.coroutines.flow.Flow -import org.jcnc.llmx.core.spi.entities.request.ChatRequest -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart /** @@ -20,4 +20,11 @@ interface BaiLianModelService { * @return 返回一个Flow流，包含部分聊天响应，允许逐步处理和消费响应 */ fun streamChat(request: ChatRequest): Flow + /** + * 使用流式多模态聊天交互 + * + * @param request 聊天请求对象，包含用户输入、上下文等信息 + * @return 返回一个Flow流，包含部分聊天响应，允许逐步处理和消费响应 + */ + fun streamMultimodalityChat(request: ChatRequest) : Flow } diff --git a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/impl/BaiLianModelServiceImpl.kt b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/impl/BaiLianModelServiceImpl.kt index 73c7d6b..f5340e6 100644 --- a/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/impl/BaiLianModelServiceImpl.kt +++ b/llmx-impl/llmx-impl-bailian/src/main/kotlin/org/jcnc/llmx/impl/baiLian/service/impl/BaiLianModelServiceImpl.kt @@ -4,11 +4,13 @@ import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper import com.gewuyou.forgeboot.core.extension.log import kotlinx.coroutines.flow.Flow -import org.jcnc.llmx.core.spi.entities.request.ChatRequest -import org.jcnc.llmx.core.spi.entities.response.ChatResponsePart -import org.jcnc.llmx.core.spi.entities.response.Usage +import org.jcnc.llmx.core.common.entities.request.ChatRequest +import org.jcnc.llmx.core.common.entities.response.ChatResponsePart +import org.jcnc.llmx.core.common.entities.response.Usage + import org.jcnc.llmx.impl.baiLian.adapter.DashScopeAdapter import org.jcnc.llmx.impl.baiLian.config.entities.DashScopeProperties +import org.jcnc.llmx.impl.baiLian.entities.response.DashScopeResponse import org.jcnc.llmx.impl.baiLian.service.BaiLianModelService import org.springframework.stereotype.Service @@ -123,4 +125,75 @@ class BaiLianModelServiceImpl( } ) } + + /** + * 使用流式多模态聊天交互 + * + * @param request 聊天请求对象，包含用户输入、上下文等信息 + * @return 返回一个Flow流，包含部分聊天响应，允许逐步处理和消费响应 + */ + override fun streamMultimodalityChat(request: ChatRequest): Flow { + // 构造请求URL + val url = dashScopeProperties.multimodalityUrl + log.info("请求URL: $url") + // 构造请求头，包括授权信息和内容类型 + val headers = mapOf( + "Authorization" to "Bearer ${dashScopeProperties.apiKey}", + "Content-Type" to "application/json", + "X-DashScope-SSE" to "enable" + ) + log.info("请求头: $headers") + // 构造输入参数，主要包括用户的message + val inputMap = mutableMapOf("messages" to request.messages) + // 构造参数地图，包括是否包含思考、模型ID、增量输出等 + val parametersMap = mutableMapOf( + "incremental_output" to true, + ) + // 构造请求体，包括输入参数、构造的参数地图和调试信息 + val body = mapOf( + "model" to request.model, + "input" to inputMap, + "parameters" to parametersMap, + ) + // 发送流式聊天请求，并处理响应 + return dashScopeAdapter.sendStreamChat( + url, headers, body, + { json: String -> + val response = objectMapper.readValue(json, DashScopeResponse::class.java) + val choices = response + .output + ?.choices + ?.getOrNull(0) + // 提取输出文本 + val text = choices + ?.message + ?.content + ?.getOrNull(0) + ?.text + ?: "" + // 判断是否完成（finish_reason 通常为 "stop"，但你这里是字符串 "null"） + val finishReason = choices?.finishReason + val done = finishReason == "stop" + // 提取使用情况 + val usage = response.usage + val promptTokens = usage?.inputTokens ?: 0 + val completionTokens = usage?.outputTokens ?: 0 + val totalTokens = usage?.totalTokens ?: 0 + ChatResponsePart( + content = text, + done = done, + usage = Usage( + promptTokens = promptTokens, + completionTokens = completionTokens, + totalTokens = totalTokens + ), + other = mapOf( + "request_id" to (response.requestId ?: ""), + "model" to request.model, + "response" to json + ) + ) + } + ) + } } diff --git a/llmx-impl/llmx-impl-bailian/src/main/resources/application-dev.yml b/llmx-impl/llmx-impl-bailian/src/main/resources/application-dev.yml index dd757e1..47077e1 100644 --- a/llmx-impl/llmx-impl-bailian/src/main/resources/application-dev.yml +++ b/llmx-impl/llmx-impl-bailian/src/main/resources/application-dev.yml @@ -22,3 +22,4 @@ aliyun: api-key: sk-78af4dd964a94f4cb373851064dbdc12 # API密钥 app-id: 3fae0bbab2e54a90a37aa02cd12dd62c # 应用ID base-url: https://dashscope.aliyuncs.com/api/v1/apps/ # 基础API URL + multimodality-url: https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation diff --git a/llmx-impl/llmx-impl-bailian/src/main/resources/application-test.yml b/llmx-impl/llmx-impl-bailian/src/main/resources/application-test.yml index c1d931f..27c53dc 100644 --- a/llmx-impl/llmx-impl-bailian/src/main/resources/application-test.yml +++ b/llmx-impl/llmx-impl-bailian/src/main/resources/application-test.yml @@ -18,3 +18,4 @@ aliyun: api-key: sk-78af4dd964a94f4cb373851064dbdc12 # API密钥 app-id: 3fae0bbab2e54a90a37aa02cd12dd62c # 应用ID base-url: https://dashscope.aliyuncs.com/api/v1/apps/ # 基础API URL + multimodality-url: https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation diff --git a/llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/Base64Image.java b/llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/Base64Image.java new file mode 100644 index 0000000..78420fe --- /dev/null +++ b/llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/Base64Image.java @@ -0,0 +1,21 @@ +package com.gewuyou.bailian.util; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Base64; + +/** + * Base 64 图像 + * + * @author gewuyou + * @since 2025-05-08 18:56:02 + */ +public class Base64Image { + public static void main(String[] args) throws Exception { + byte[] imageBytes = Files.readAllBytes(Paths.get("F:/gewuyou/Project/Idea/demo-llm/src/main/resources/2025俏春装/10147009251.jpg")); + String base64Image = Base64.getEncoder().encodeToString(imageBytes); + String imagePayload = "data:image/jpeg;base64," + base64Image; + + System.out.println(imagePayload); // 你可以把它放到 JSON 中使用 + } +} diff --git a/llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/DashScopeLocalImageClient.java b/llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/DashScopeLocalImageClient.java new file mode 100644 index 0000000..6d3536f --- /dev/null +++ b/llmx-impl/llmx-impl-bailian/src/test/java/com/gewuyou/bailian/util/DashScopeLocalImageClient.java @@ -0,0 +1,85 @@ +package com.gewuyou.bailian.util; + + + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.util.Base64; +/** + * DashScopeLocalImageClient + * + * @author gewuyou + * @since 2025-05-08 18:57:58 + */ +public class DashScopeLocalImageClient { + + public static void main(String[] args) throws Exception { + // Step 1: 读取本地图片并转成 base64 + String imagePath = "F:/gewuyou/Project/Idea/demo-llm/src/main/resources/2025俏春装/10147009251.jpg"; + byte[] imageBytes = Files.readAllBytes(new File(imagePath).toPath()); + String base64Image = Base64.getEncoder().encodeToString(imageBytes); + String imageDataUrl = "data:image/jpeg;base64," + base64Image; + System.out.println(imageDataUrl); + // Step 2: 构造请求 JSON + String jsonBody = """ + { + "model": "qwen-vl-max-latest", + "input": { + "messages": [ + { + "role": "system", + "content": [ + { "text": "You are a helpful assistant." } + ] + }, + { + "role": "user", + "content": [ + { "image": "%s" }, + { "text": "请给这张图打标签" } + ] + } + ] + }, + "parameters": { + "incremental_output": true + } + } + """.formatted(imageDataUrl.replace("\n", "")); + + // Step 3: 发送 POST 请求 + URL url = new URL("https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + + connection.setRequestMethod("POST"); + connection.setRequestProperty("Authorization", "Bearer sk-78af4dd964a94f4cb373851064dbdc12"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + + try (OutputStream os = connection.getOutputStream()) { + os.write(jsonBody.getBytes()); + os.flush(); + } + + // Step 4: 读取响应 + int responseCode = connection.getResponseCode(); + System.out.println("Response Code: " + responseCode); + + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(responseCode == 200 ? + connection.getInputStream() : connection.getErrorStream(), "GBK"))) { + + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + } + + connection.disconnect(); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index e3f326d..04461a1 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -6,15 +6,25 @@ include( "llmx-core", ":llmx-core:llmx-core-service", ":llmx-core:llmx-core-spi", + ":llmx-core:llmx-core-api", + ":llmx-core:llmx-core-common", ) project(":llmx-core:llmx-core-service").name = "llmx-core-service" project(":llmx-core:llmx-core-spi").name = "llmx-core-spi" +project(":llmx-core:llmx-core-api").name = "llmx-core-api" +project(":llmx-core:llmx-core-common").name = "llmx-core-common" include( "llmx-impl", - "llmx-impl:llmx-impl-bailian", + ":llmx-impl:llmx-impl-bailian", // "llmx-impl:llmx-impl-openAi", ) project(":llmx-impl:llmx-impl-bailian").name = "llmx-impl-bailian" -//project(":llmx-impl:llmx-impl-openAi").name = "llmx-impl-openAi" \ No newline at end of file +//project(":llmx-impl:llmx-impl-openAi").name = "llmx-impl-openAi" + +include( + "llmx-app", + ":llmx-app:llmx-app-multimodality" +) +project(":llmx-app:llmx-app-multimodality").name = "llmx-app-multimodality" \ No newline at end of file