添加题目同步到知识库的功能

.run/docker.run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="docker" type="docker-deploy" factoryName="docker-compose.yml" server-name="Docker">
+  <configuration default="false" name=".script/integration_test_compose.yml: Compose 部署" type="docker-deploy" factoryName="docker-compose.yml" server-name="Docker">
     <deployment type="docker-compose.yml">
       <settings>
         <option name="envFilePath" value="" />

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/google/wire v0.6.0
 	github.com/gotomicro/ego v1.1.19
 	github.com/lithammer/shortuuid/v4 v4.0.0
+	github.com/lukasjarosch/go-docx v0.5.0
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/pkg/errors v0.9.1
 	github.com/redis/go-redis/v9 v9.4.0
@@ -24,7 +25,7 @@ require (
 	github.com/wechatpay-apiv3/wechatpay-go v0.2.18
 	github.com/yankeguo/zhipu v0.1.0
 	go.uber.org/mock v0.3.0
-	golang.org/x/sync v0.6.0
+	golang.org/x/sync v0.7.0
 	gorm.io/gorm v1.25.1
 )
 
@@ -64,6 +65,7 @@ require (
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/cel-go v0.11.3 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gotomicro/logrotate v0.0.0-20211108034117-46d53eedc960 // indirect
@@ -123,11 +125,11 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/arch v0.7.0 // indirect
-	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/grpc v1.58.1 // indirect

go.sum

@@ -275,8 +275,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -455,6 +455,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c=
 github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y=
+github.com/lukasjarosch/go-docx v0.5.0 h1:4vU+gJ4WMdqwRvRVFF+XMw3rPfUGSXlToPJIX3mHQsQ=
+github.com/lukasjarosch/go-docx v0.5.0/go.mod h1:ka/NZgDIJId48vMvcfWfduVTY7uV0/f8EgsmCjuS9X0=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
@@ -747,8 +749,9 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -819,6 +822,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -830,8 +834,9 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -850,8 +855,9 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -916,8 +922,9 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -943,8 +950,9 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

internal/ai/internal/domain/knowledge_base.go

@@ -0,0 +1,24 @@
+package domain
+
+// 知识库的相关对象
+
+// KnowledgeBaseFile 知识库文件
+type KnowledgeBaseFile struct {
+	Biz   string
+	BizID int64
+	// 平台
+	Platform string
+	// 文件名
+	Name string
+	// 文件内容
+	Data []byte
+	// 用途
+	Type            string
+	FileID          string
+	KnowledgeBaseID string
+}
+
+const (
+	RepositoryBaseTypeRetrieval = "retrieval"
+	RepositoryBaseTypeFineTune  = "finetune"
+)

internal/ai/internal/domain/llm.go

@@ -65,6 +65,7 @@ type BizConfig struct {
 	// 这里一般使用 %s
 	// 后续考虑 key value 的形式
 	PromptTemplate string
+	Utime          int64
 }
 
 type LLMCredit struct {

internal/ai/internal/event/consumer.go

@@ -0,0 +1,73 @@
+package event
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"github.com/ecodeclub/mq-api"
+	"github.com/ecodeclub/webook/internal/ai/internal/domain"
+	"github.com/ecodeclub/webook/internal/ai/internal/service/llm/knowledge_base"
+	"github.com/gotomicro/ego/core/elog"
+)
+
+type KnowledgeBaseConsumer struct {
+	svc      knowledge_base.RepositoryBaseSvc
+	consumer mq.Consumer
+	logger   *elog.Component
+}
+
+func NewKnowledgeBaseConsumer(svc knowledge_base.RepositoryBaseSvc, q mq.MQ) (*KnowledgeBaseConsumer, error) {
+	groupID := "knowledge_base_group"
+	consumer, err := q.Consumer(KnowledgeBaseUploadTopic, groupID)
+	if err != nil {
+		return nil, err
+	}
+	return &KnowledgeBaseConsumer{
+		svc:      svc,
+		consumer: consumer,
+		logger:   elog.DefaultLogger,
+	}, nil
+}
+
+func (k *KnowledgeBaseConsumer) Consume(ctx context.Context) error {
+	msg, err := k.consumer.Consume(ctx)
+	if err != nil {
+		return fmt.Errorf("获取消息失败: %w", err)
+	}
+
+	var evt KnowledgeBaseUploadEvent
+	err = json.Unmarshal(msg.Value, &evt)
+	if err != nil {
+		return fmt.Errorf("解析消息失败: %w", err)
+	}
+	log.Println("xxxxx", evt)
+	err = k.svc.UploadFile(ctx, domain.KnowledgeBaseFile{
+		Biz:             evt.Biz,
+		BizID:           evt.BizID,
+		Name:            evt.Name,
+		Data:            evt.Data,
+		Type:            evt.Type,
+		KnowledgeBaseID: evt.KnowledgeBaseID,
+	})
+	if err != nil {
+		return fmt.Errorf("上传文件到知识库失败 %w", err)
+	}
+	return nil
+}
+
+func (k *KnowledgeBaseConsumer) Start(ctx context.Context) {
+	go func() {
+		for {
+			err := k.Consume(ctx)
+			if err != nil {
+				k.logger.Error("同步事件失败", elog.FieldErr(err))
+			}
+		}
+	}()
+}
+
+func (k *KnowledgeBaseConsumer) Stop(_ context.Context) error {
+	return k.consumer.Close()
+}

internal/ai/internal/event/event.go

@@ -0,0 +1,18 @@
+package event
+
+// KnowledgeBaseUploadEvent 知识库上传事件
+type KnowledgeBaseUploadEvent struct {
+	Biz   string `json:"biz"`
+	BizID int64  `json:"bizID"`
+	// 文件名
+	Name string `json:"name"`
+	// 文件内容
+	Data []byte `json:"data"`
+	// 用途
+	Type            string `json:"type"`
+	KnowledgeBaseID string `json:"knowledgeBaseID"`
+}
+
+const (
+	KnowledgeBaseUploadTopic = "knowledge_base_upload_topic"
+)

internal/ai/internal/integration/llm_config_test.go

@@ -36,7 +36,7 @@ func (s *ConfigSuite) SetupSuite() {
 	err := dao.InitTables(db)
 	s.NoError(err)
 	// 先插入 BizConfig
-	mou, err := startup.InitModule(s.db, nil, &credit.Module{})
+	mou, err := startup.InitModule(s.db, nil, nil, &credit.Module{}, nil)
 	require.NoError(s.T(), err)
 	s.adminHandler = mou.AdminHandler
 	econf.Set("server", map[string]any{"contextTimeout": "1s"})
@@ -190,6 +190,7 @@ func (s *ConfigSuite) TestConfig_List() {
 			SystemPrompt:   "test_prompt",
 			PromptTemplate: "test_template",
 			KnowledgeId:    "test_knowledge",
+			Utime:          int64(i),
 		}
 		configs = append(configs, cfg)
 	}
@@ -252,6 +253,7 @@ func (s *ConfigSuite) Test_Detail() {
 				SystemPrompt:   "testPrompt",
 				PromptTemplate: "testTemplate",
 				KnowledgeId:    "testKnowledgeId",
+				Utime:          22,
 			},
 		},
 	}
@@ -287,6 +289,7 @@ func getWantConfigs() []web.Config {
 			SystemPrompt:   "test_prompt",
 			PromptTemplate: "test_template",
 			KnowledgeId:    "test_knowledge",
+			Utime:          int64(i),
 		}
 		configs = append(configs, cfg)
 	}

internal/ai/internal/integration/llm_knowledge_base_test.go

@@ -0,0 +1,44 @@
+package integration
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ecodeclub/webook/internal/ai/internal/domain"
+	"github.com/ecodeclub/webook/internal/ai/internal/integration/startup"
+	testioc "github.com/ecodeclub/webook/internal/test/ioc"
+	"github.com/stretchr/testify/require"
+)
+
+func TestKnowledgeBaseTest(t *testing.T) {
+	t.Skip("替换api.key运行")
+
+	db := testioc.InitDB()
+	startup.InitTableOnce(db)
+	baseSvc := startup.InitKnowledgeBaseSvc(db, "api.key")
+	testCases := []struct {
+		name string
+		file domain.KnowledgeBaseFile
+	}{
+		{
+			name: "正常上传",
+			file: domain.KnowledgeBaseFile{
+				Biz:   "question",
+				BizID: 4,
+				Name:  "question4",
+				Type:  domain.RepositoryBaseTypeRetrieval,
+				Data:  []byte("test999999"),
+				// 这个也要修改
+				KnowledgeBaseID: "1863183941318684672",
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// 执行测试
+			err := baseSvc.UploadFile(context.Background(), tc.file)
+			require.NoError(t, err)
+		})
+	}
+
+}