Skip to content

Commit

Permalink
添加题目同步到知识库的功能
Browse files Browse the repository at this point in the history
  • Loading branch information
juniaoshaonian committed Dec 5, 2024
1 parent f11be44 commit dec2798
Show file tree
Hide file tree
Showing 40 changed files with 858 additions and 72 deletions.
2 changes: 1 addition & 1 deletion .run/docker.run.xml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="docker" type="docker-deploy" factoryName="docker-compose.yml" server-name="Docker">
<configuration default="false" name=".script/integration_test_compose.yml: Compose Deployment" type="docker-deploy" factoryName="docker-compose.yml" server-name="Docker">
<deployment type="docker-compose.yml">
<settings>
<option name="envFilePath" value="" />
Expand Down
Binary file added doc/template.docx
Binary file not shown.
11 changes: 6 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/google/wire v0.6.0
github.com/gotomicro/ego v1.1.19
github.com/lithammer/shortuuid/v4 v4.0.0
github.com/lukasjarosch/go-docx v0.5.0
github.com/olivere/elastic/v7 v7.0.32
github.com/pkg/errors v0.9.1
github.com/redis/go-redis/v9 v9.4.0
Expand All @@ -24,7 +25,7 @@ require (
github.com/wechatpay-apiv3/wechatpay-go v0.2.18
github.com/yankeguo/zhipu v0.1.0
go.uber.org/mock v0.3.0
golang.org/x/sync v0.6.0
golang.org/x/sync v0.7.0
gorm.io/gorm v1.25.1
)

Expand Down Expand Up @@ -123,11 +124,11 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/arch v0.7.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/grpc v1.58.1 // indirect
Expand Down
18 changes: 13 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c=
github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y=
github.com/lukasjarosch/go-docx v0.5.0 h1:4vU+gJ4WMdqwRvRVFF+XMw3rPfUGSXlToPJIX3mHQsQ=
github.com/lukasjarosch/go-docx v0.5.0/go.mod h1:ka/NZgDIJId48vMvcfWfduVTY7uV0/f8EgsmCjuS9X0=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
Expand Down Expand Up @@ -747,8 +749,9 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand Down Expand Up @@ -819,6 +822,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
Expand All @@ -830,8 +834,9 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand All @@ -850,8 +855,9 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -916,8 +922,9 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand All @@ -943,8 +950,9 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
24 changes: 24 additions & 0 deletions internal/ai/internal/domain/knowledge_base.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package domain

// 知识库的相关对象

// KnowledgeBaseFile 知识库文件
type KnowledgeBaseFile struct {
Biz string
BizID int64
// 平台
Platform string
// 文件名
Name string
// 文件内容
Data []byte
// 用途
Type string
FileID string
KnowledgeBaseID string
}

const (
RepositoryBaseTypeRetrieval = "retrieval"
RepositoryBaseTypeFineTune = "finetune"
)
1 change: 1 addition & 0 deletions internal/ai/internal/domain/llm.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ type BizConfig struct {
// 这里一般使用 %s
// 后续考虑 key value 的形式
PromptTemplate string
Utime int64
}

type LLMCredit struct {
Expand Down
Binary file not shown.
5 changes: 4 additions & 1 deletion internal/ai/internal/integration/llm_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func (s *ConfigSuite) SetupSuite() {
err := dao.InitTables(db)
s.NoError(err)
// 先插入 BizConfig
mou, err := startup.InitModule(s.db, nil, &credit.Module{})
mou, err := startup.InitModule(s.db, nil, nil, &credit.Module{})
require.NoError(s.T(), err)
s.adminHandler = mou.AdminHandler
econf.Set("server", map[string]any{"contextTimeout": "1s"})
Expand Down Expand Up @@ -190,6 +190,7 @@ func (s *ConfigSuite) TestConfig_List() {
SystemPrompt: "test_prompt",
PromptTemplate: "test_template",
KnowledgeId: "test_knowledge",
Utime: int64(i),
}
configs = append(configs, cfg)
}
Expand Down Expand Up @@ -252,6 +253,7 @@ func (s *ConfigSuite) Test_Detail() {
SystemPrompt: "testPrompt",
PromptTemplate: "testTemplate",
KnowledgeId: "testKnowledgeId",
Utime: 22,
},
},
}
Expand Down Expand Up @@ -287,6 +289,7 @@ func getWantConfigs() []web.Config {
SystemPrompt: "test_prompt",
PromptTemplate: "test_template",
KnowledgeId: "test_knowledge",
Utime: int64(i),
}
configs = append(configs, cfg)
}
Expand Down
44 changes: 44 additions & 0 deletions internal/ai/internal/integration/llm_knowledge_base_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package integration

import (
"context"
"testing"

"github.com/ecodeclub/webook/internal/ai/internal/domain"
"github.com/ecodeclub/webook/internal/ai/internal/integration/startup"
testioc "github.com/ecodeclub/webook/internal/test/ioc"
"github.com/stretchr/testify/require"
)

func TestKnowledgeBaseTest(t *testing.T) {
t.Skip("替换api.key运行")

db := testioc.InitDB()
startup.InitTableOnce(db)
baseSvc := startup.InitKnowledgeBaseSvc(db, "api.key")
testCases := []struct {
name string
file domain.KnowledgeBaseFile
}{
{
name: "正常上传",
file: domain.KnowledgeBaseFile{
Biz: "question",
BizID: 4,
Name: "question4",
Type: domain.RepositoryBaseTypeRetrieval,
Data: []byte("test999999"),
// 这个也要修改
KnowledgeBaseID: "1863183941318684672",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// 执行测试
err := baseSvc.UploadFile(context.Background(), tc.file)
require.NoError(t, err)
})
}

}
6 changes: 3 additions & 3 deletions internal/ai/internal/integration/llm_service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -489,7 +489,7 @@ func (s *LLMServiceSuite) TestService() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
mockHdl, mockCredit := tc.before(t, ctrl)
mou, err := startup.InitModule(s.db, mockHdl, &credit.Module{Svc: mockCredit})
mou, err := startup.InitModule(s.db, mockHdl, nil, &credit.Module{Svc: mockCredit})
require.NoError(t, err)
resp, err := mou.Svc.Invoke(ctx, tc.req)
tc.assertFunc(t, err)
Expand Down Expand Up @@ -595,7 +595,7 @@ func (s *LLMServiceSuite) TestHandler_Ask() {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockHdl, mockCredit := tc.before(t, ctrl)
mou, err := startup.InitModule(s.db, mockHdl, &credit.Module{Svc: mockCredit})
mou, err := startup.InitModule(s.db, mockHdl, nil, &credit.Module{Svc: mockCredit})
require.NoError(t, err)
req, err := http.NewRequest(http.MethodPost,
"/ai/ask", iox.NewJSONReader(tc.req))
Expand Down Expand Up @@ -701,7 +701,7 @@ func (s *LLMServiceSuite) TestHandler_AnalysisJD() {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockHdl, mockCredit := tc.before(t, ctrl)
mou, err := startup.InitModule(s.db, mockHdl, &credit.Module{Svc: mockCredit})
mou, err := startup.InitModule(s.db, mockHdl, nil, &credit.Module{Svc: mockCredit})
require.NoError(t, err)
req, err := http.NewRequest(http.MethodPost,
"/ai/analysis_jd", iox.NewJSONReader(tc.req))
Expand Down
16 changes: 16 additions & 0 deletions internal/ai/internal/integration/startup/wire.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ package startup
import (
"sync"

"github.com/ecodeclub/webook/internal/ai/internal/service/llm/knowledge_base/zhipu"

"github.com/ecodeclub/webook/internal/ai/internal/service/llm/knowledge_base"

"github.com/ecodeclub/webook/internal/ai/internal/service"
"github.com/ecodeclub/webook/internal/ai/internal/web"

Expand All @@ -28,6 +32,7 @@ import (

func InitModule(db *egorm.Component,
hdl *hdlmocks.MockHandler,
baseSvc knowledge_base.RepositoryBaseSvc,
creditSvc *credit.Module) (*ai.Module, error) {
wire.Build(
llm.NewLLMService,
Expand Down Expand Up @@ -57,6 +62,17 @@ func InitModule(db *egorm.Component,
return new(ai.Module), nil
}

func InitKnowledgeBaseSvc(db *egorm.Component, apikey string) knowledge_base.RepositoryBaseSvc {
knowledgeDao := dao.NewKnowledgeBaseDAO(db)
knowledgeRepo := repository.NewKnowledgeBaseRepo(knowledgeDao)
// 将智谱对应的apikey写到环境变量
knowledgeSvc, err := zhipu.NewKnowledgeBase(apikey, knowledgeRepo)
if err != nil {
panic(err)
}
return knowledgeSvc
}

func InitRootHandler(common []handler.Builder, hdl *hdlmocks.MockHandler) handler.Handler {
return handler.NewCompositionHandler(common, hdl)
}
Expand Down
22 changes: 18 additions & 4 deletions internal/ai/internal/integration/startup/wire_gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit dec2798

Please sign in to comment.