From 09393aed8f4bcdbce29ea0e470a52e66cb165470 Mon Sep 17 00:00:00 2001 From: "fox.cpp" Date: Sun, 11 Jul 2021 21:42:19 +0300 Subject: [PATCH] Refactor imapsql ExternalStore to use modules Closes #303 --- .mkdocs.yml | 1 + docs/man/maddy-blob.5.scd | 41 +++++++++ docs/man/maddy-storage.5.scd | 16 ++-- framework/module/blob_store.go | 30 +++++++ internal/storage/blob/fs/fs.go | 89 +++++++++++++++++++ .../storage/imapsql/external_blob_store.go | 59 ++++++++++++ internal/storage/imapsql/imapsql.go | 43 ++++----- maddy.go | 3 + 8 files changed, 255 insertions(+), 27 deletions(-) create mode 100644 docs/man/maddy-blob.5.scd create mode 100644 framework/module/blob_store.go create mode 100644 internal/storage/blob/fs/fs.go create mode 100644 internal/storage/imapsql/external_blob_store.go diff --git a/.mkdocs.yml b/.mkdocs.yml index e159dbe0..e9b5c077 100644 --- a/.mkdocs.yml +++ b/.mkdocs.yml @@ -29,6 +29,7 @@ nav: - man/_generated_maddy.1.md - man/_generated_maddy.5.md - man/_generated_maddy-auth.5.md + - man/_generated_maddy-blob.5.md - man/_generated_maddy-config.5.md - man/_generated_maddy-filters.5.md - man/_generated_maddy-imap.5.md diff --git a/docs/man/maddy-blob.5.scd b/docs/man/maddy-blob.5.scd new file mode 100644 index 00000000..712c7a4c --- /dev/null +++ b/docs/man/maddy-blob.5.scd @@ -0,0 +1,41 @@ +maddy-blob(5) "maddy mail server" "maddy reference documentation" + +; TITLE Message blob storage + +Some IMAP storage backends support pluggable message storage that allows +message contents to be stored separately from IMAP index. + +Modules described in this page are what can be used with such storage backends. +In most cases they have to be specified using the 'msg_store' directive, like +this: +``` +storage.imapsql local_mailboxes { + msg_store fs /var/lib/email +} +``` + +Unless explicitly configured, storage backends with pluggable storage will +store messages in state_dir/messages (e.g. /var/lib/maddy/messages) FS +directory. + +# FS directory storage (storage.blob.fs) + +This module stores message bodies in a file system directory. + +``` +storage.blob.fs { + root +} +``` +``` +storage.blob.fs +``` + +## Configuration directives + +*Syntax:* root _path_ ++ +*Default:* not set + +Path to the FS directory. Must be readable and writable by the server process. +If it does not exist - it will be created (parent directory should be writable +for this). Relative paths are interpreted relatively to server state directory. \ No newline at end of file diff --git a/docs/man/maddy-storage.5.scd b/docs/man/maddy-storage.5.scd index 7cd3d56a..bcdee04c 100644 --- a/docs/man/maddy-storage.5.scd +++ b/docs/man/maddy-storage.5.scd @@ -21,12 +21,11 @@ created. # SQL-based database module (storage.imapsql) -The imapsql module implements unified database for IMAP index and message +The imapsql module implements database for IMAP index and message metadata using SQL-based relational database. -Message contents are stored in an "external store", currently the only -supported "external store" is a filesystem directory, used by default. -By default, all messages are stored in StateDirectory/messages under random IDs. +Message contents are stored in an "external store" defined by msg_store +directive. By default this is a file system directory under /var/lib/maddy. Supported RDBMS: - SQLite 3.25.0 @@ -40,6 +39,7 @@ PRECIS UsernameCaseMapped profile. storage.imapsql { driver sqlite3 dsn imapsql.db + msg_store fs messages/ } ``` @@ -88,10 +88,12 @@ For PostgreSQL: https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parame Should be specified either via an argument or via this directive. -*Syntax*: fsstore _directory_ ++ -*Default*: messages/ +*Syntax*: msg_store _store_ ++ +*Default*: fs messages/ -Directory to store message contents in. +Module to use for message bodies storage. + +See *maddy-blob*(5) for details. *Syntax*: ++ compression off ++ diff --git a/framework/module/blob_store.go b/framework/module/blob_store.go new file mode 100644 index 00000000..bd30ed1a --- /dev/null +++ b/framework/module/blob_store.go @@ -0,0 +1,30 @@ +package module + +import ( + "errors" + "io" +) + +type Blob interface { + Sync() error + io.Reader + io.Writer + io.Closer +} + +var ErrNoSuchBlob = errors.New("blob_store: no such object") + +// BlobStore is the interface used by modules providing large binary object +// storage. +type BlobStore interface { + Create(key string) (Blob, error) + + // Open returns the reader for the object specified by + // passed key. + // + // If no such object exists - ErrNoSuchBlob is returned. + Open(key string) (io.ReadCloser, error) + + // Delete removes a set of keys from store. Non-existent keys are ignored. + Delete(keys []string) error +} diff --git a/internal/storage/blob/fs/fs.go b/internal/storage/blob/fs/fs.go new file mode 100644 index 00000000..3304c8d5 --- /dev/null +++ b/internal/storage/blob/fs/fs.go @@ -0,0 +1,89 @@ +package fs + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/foxcpp/maddy/framework/config" + "github.com/foxcpp/maddy/framework/module" +) + +// FSStore struct represents directory on FS used to store blobs. +type FSStore struct { + instName string + root string +} + +func New(_, instName string, _, inlineArgs []string) (module.Module, error) { + switch len(inlineArgs) { + case 0: + return &FSStore{instName: instName}, nil + case 1: + return &FSStore{instName: instName, root: inlineArgs[0]}, nil + default: + return nil, fmt.Errorf("storage.blob.fs: 1 or 0 arguments expected") + } +} + +func (s FSStore) Name() string { + return "storage.blob.fs" +} + +func (s FSStore) InstanceName() string { + return s.instName +} + +func (s *FSStore) Init(cfg *config.Map) error { + cfg.String("root", false, false, s.root, &s.root) + if _, err := cfg.Process(); err != nil { + return err + } + + if s.root == "" { + return config.NodeErr(cfg.Block, "storage.blob.fs: directory not set") + } + + if err := os.MkdirAll(s.root, os.ModeDir|os.ModePerm); err != nil { + return err + } + + return nil +} + +func (s *FSStore) Open(key string) (io.ReadCloser, error) { + f, err := os.Open(filepath.Join(s.root, key)) + if err != nil { + if os.IsNotExist(err) { + return nil, module.ErrNoSuchBlob + } + return nil, err + } + return f, nil +} + +func (s *FSStore) Create(key string) (module.Blob, error) { + f, err := os.Create(filepath.Join(s.root, key)) + if err != nil { + return nil, err + } + return f, nil +} + +func (s *FSStore) Delete(keys []string) error { + for _, key := range keys { + if err := os.Remove(filepath.Join(s.root, key)); err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + } + return nil +} + +func init() { + var _ module.BlobStore = &FSStore{} + module.Register(FSStore{}.Name(), New) +} diff --git a/internal/storage/imapsql/external_blob_store.go b/internal/storage/imapsql/external_blob_store.go new file mode 100644 index 00000000..719b5b6b --- /dev/null +++ b/internal/storage/imapsql/external_blob_store.go @@ -0,0 +1,59 @@ +package imapsql + +import ( + "io" + + imapsql "github.com/foxcpp/go-imap-sql" + "github.com/foxcpp/maddy/framework/module" +) + +type ExtBlob struct { + io.ReadCloser +} + +func (e ExtBlob) Sync() error { + panic("not implemented") +} + +func (e ExtBlob) Write(p []byte) (n int, err error) { + panic("not implemented") +} + +type ExtBlobStore struct { + base module.BlobStore +} + +func (e ExtBlobStore) Create(key string) (imapsql.ExtStoreObj, error) { + blob, err := e.base.Create(key) + if err != nil { + return nil, imapsql.ExternalError{ + NonExistent: err == module.ErrNoSuchBlob, + Key: key, + Err: err, + } + } + return blob, nil +} + +func (e ExtBlobStore) Open(key string) (imapsql.ExtStoreObj, error) { + blob, err := e.base.Open(key) + if err != nil { + return nil, imapsql.ExternalError{ + NonExistent: err == module.ErrNoSuchBlob, + Key: key, + Err: err, + } + } + return ExtBlob{ReadCloser: blob}, nil +} + +func (e ExtBlobStore) Delete(keys []string) error { + err := e.base.Delete(keys) + if err != nil { + return imapsql.ExternalError{ + Key: "", + Err: err, + } + } + return nil +} diff --git a/internal/storage/imapsql/imapsql.go b/internal/storage/imapsql/imapsql.go index 95d57838..9c02aa35 100644 --- a/internal/storage/imapsql/imapsql.go +++ b/internal/storage/imapsql/imapsql.go @@ -31,7 +31,6 @@ import ( "encoding/hex" "errors" "fmt" - "os" "path/filepath" "runtime/debug" "strconv" @@ -103,12 +102,13 @@ func New(_, instName string, _, inlineArgs []string) (module.Module, error) { func (store *Storage) Init(cfg *config.Map) error { var ( - driver string - dsn []string - fsstoreLocation string - appendlimitVal = -1 - compression []string - authNormalize string + driver string + dsn []string + appendlimitVal = -1 + compression []string + authNormalize string + + blobStore module.BlobStore ) opts := imapsql.Opts{ @@ -118,14 +118,22 @@ func (store *Storage) Init(cfg *config.Map) error { } cfg.String("driver", false, false, store.driver, &driver) cfg.StringList("dsn", false, false, store.dsn, &dsn) - cfg.Custom("fsstore", false, false, func() (interface{}, error) { - return "messages", nil + cfg.Callback("fsstore", func(m *config.Map, node config.Node) error { + store.Log.Msg("'fsstore' directive is deprecated, use 'msg_store fs' instead") + return modconfig.ModuleFromNode("storage.blob", append([]string{"fs"}, node.Args...), + node, m.Globals, &blobStore) + }) + cfg.Custom("msg_store", false, false, func() (interface{}, error) { + var store module.BlobStore + err := modconfig.ModuleFromNode("storage.blob", []string{"fs", "messages"}, + config.Node{}, nil, &store) + return store, err }, func(m *config.Map, node config.Node) (interface{}, error) { - if len(node.Args) != 1 { - return nil, config.NodeErr(node, "expected 0 or 1 arguments") - } - return node.Args[0], nil - }, &fsstoreLocation) + var store module.BlobStore + err := modconfig.ModuleFromNode("storage.blob", node.Args, + node, m.Globals, &store) + return store, err + }, &blobStore) cfg.StringList("compression", false, false, []string{"off"}, &compression) cfg.DataSize("appendlimit", false, false, 32*1024*1024, &appendlimitVal) cfg.Bool("debug", true, false, &store.Log.Debug) @@ -195,11 +203,6 @@ func (store *Storage) Init(cfg *config.Map) error { dsnStr := strings.Join(dsn, " ") - if err := os.MkdirAll(fsstoreLocation, os.ModeDir|os.ModePerm); err != nil { - return err - } - extStore := &imapsql.FSStore{Root: fsstoreLocation} - if len(compression) != 0 { switch compression[0] { case "zstd", "lz4": @@ -222,7 +225,7 @@ func (store *Storage) Init(cfg *config.Map) error { } } - store.Back, err = imapsql.New(driver, dsnStr, extStore, opts) + store.Back, err = imapsql.New(driver, dsnStr, ExtBlobStore{base: blobStore}, opts) if err != nil { return fmt.Errorf("imapsql: %s", err) } diff --git a/maddy.go b/maddy.go index 620ee829..54257a70 100644 --- a/maddy.go +++ b/maddy.go @@ -61,6 +61,7 @@ import ( _ "github.com/foxcpp/maddy/internal/imap_filter/command" _ "github.com/foxcpp/maddy/internal/modify" _ "github.com/foxcpp/maddy/internal/modify/dkim" + _ "github.com/foxcpp/maddy/internal/storage/blob/fs" _ "github.com/foxcpp/maddy/internal/storage/imapsql" _ "github.com/foxcpp/maddy/internal/table" _ "github.com/foxcpp/maddy/internal/target/queue" @@ -344,6 +345,8 @@ func RegisterModules(globals map[string]interface{}, nodes []config.Node) (endpo } module.RegisterAlias(alias, instName) } + + log.Debugf("%v:%v: register config block %v %v", block.File, block.Line, instName, modAliases) mods = append(mods, ModInfo{Instance: inst, Cfg: block}) }