modules: tt.yaml modules/directory support list

Closes #1012

@TarantoolBot document
Title: Allow multiple directories with modules

The logic of working with configuration file tt.yaml has been updated,
now  modules/directory can be specified both as a single line and
as a list.

**Usage example**
Old behavior:
```yaml
modules:
  directory: modules
```
Possibility to specify a list of directories:
```yaml
modules:
  directory:
  - modules
  - /ext/path/modules
  - other_modules
```

If a relative path is specified, the search is performed in
the subfolders below, relative to the tt.yaml file. If an absolute
path is specified, external modules are searched according to
the specified path.

CHANGELOG.md

@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   time format.
 - `tt connect`: add new `--evaler` option to support for customizing the way
   user input is processed.
+- `tt.yaml`: allows to specify a list of modules directories.
 
 ### Changed
 

cli/cfg/dump_test.go

@@ -93,7 +93,8 @@ env:
   restart_on_failure: false
   tarantoolctl_layout: false
 modules:
-  directory: /root/modules
+  directory:
+  - /root/modules
 app:
   run_dir: var/run
   log_dir: var/log
@@ -129,7 +130,8 @@ env:
   restart_on_failure: false
   tarantoolctl_layout: false
 modules:
-  directory: %[1]s/my_modules
+  directory:
+  - %[1]s/my_modules
 app:
   run_dir: var/run
   log_dir: var/log
@@ -166,7 +168,8 @@ env:
   restart_on_failure: false
   tarantoolctl_layout: false
 modules:
-  directory: /root/modules
+  directory:
+  - /root/modules
 app:
   run_dir: var/run
   log_dir: var/log

cli/config/config.go

@@ -21,11 +21,15 @@ package config
 //  ee:
 //    credential_path: path
 
+// FieldStringArrayType a custom type used with mapstructure's hook to accept values
+// as a single string as well as a list of strings.
+type FieldStringArrayType []string
+
 // ModuleOpts is used to store all module options.
 type ModulesOpts struct {
-	// Directory is a path to directory where the external modules
+	// Directories is a list of paths to directories where the external modules
 	// are stored.
-	Directory string
+	Directories FieldStringArrayType `mapstructure:"directory" yaml:"directory"`
 }
 
 // EEOpts is used to store tarantool-ee options.

cli/configure/configure.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"syscall"
 
@@ -103,7 +104,7 @@ func getSystemAppOpts() *config.AppOpts {
 // GetDefaultCliOpts returns `CliOpts` filled with default values.
 func GetSystemCliOpts() *config.CliOpts {
 	modules := config.ModulesOpts{
-		Directory: ModulesPath,
+		Directories: []string{ModulesPath},
 	}
 	ee := config.EEOpts{
 		CredPath: "",
@@ -128,7 +129,7 @@ func GetSystemCliOpts() *config.CliOpts {
 // GetDefaultCliOpts returns `CliOpts` filled with default values.
 func GetDefaultCliOpts() *config.CliOpts {
 	modules := config.ModulesOpts{
-		Directory: ModulesPath,
+		Directories: []string{ModulesPath},
 	}
 	ee := config.EEOpts{
 		CredPath: "",
@@ -179,6 +180,24 @@ func adjustPathWithConfigLocation(filePath string, configDir string,
 	return filepath.Abs(filepath.Join(configDir, filePath))
 }
 
+func adjustListPathWithConfigLocation(listPaths []string, configDir string,
+	defaultDirName string) ([]string, error) {
+	if len(listPaths) == 0 {
+		listPaths = append(listPaths, defaultDirName)
+	}
+
+	result := make([]string, 0, len(listPaths))
+	for _, path := range listPaths {
+		path, err := adjustPathWithConfigLocation(path, configDir, defaultDirName)
+		if err != nil {
+			return result, err
+		}
+		result = append(result, path)
+
+	}
+	return result, nil
+}
+
 // resolveConfigPaths resolves all paths in config relative to specified location, and
 // sets uninitialized values to defaults.
 func updateCliOpts(cliOpts *config.CliOpts, configDir string) error {
@@ -211,8 +230,8 @@ func updateCliOpts(cliOpts *config.CliOpts, configDir string) error {
 	}
 
 	if cliOpts.Modules != nil {
-		if cliOpts.Modules.Directory, err = adjustPathWithConfigLocation(cliOpts.Modules.Directory,
-			configDir, ModulesPath); err != nil {
+		if cliOpts.Modules.Directories, err = adjustListPathWithConfigLocation(
+			cliOpts.Modules.Directories, configDir, ModulesPath); err != nil {
 			return err
 		}
 	}
@@ -227,6 +246,26 @@ func updateCliOpts(cliOpts *config.CliOpts, configDir string) error {
 	return nil
 }
 
+func decodeStringAsArrayField(from reflect.Type, to reflect.Type, value interface{}) (
+	interface{}, error) {
+	if to != reflect.TypeOf(config.FieldStringArrayType{}) || from.Kind() != reflect.String {
+		return value, nil
+	}
+	return []string{value.(string)}, nil
+}
+
+func decodeConfig(input map[string]any, cfg *config.CliOpts) error {
+	decoder_config := mapstructure.DecoderConfig{
+		Result:     cfg,
+		DecodeHook: mapstructure.ComposeDecodeHookFunc(decodeStringAsArrayField),
+	}
+	decoder, err := mapstructure.NewDecoder(&decoder_config)
+	if err != nil {
+		return err
+	}
+	return decoder.Decode(input)
+}
+
 // GetCliOpts returns Tarantool CLI options from the config file
 // located at path configurePath.
 func GetCliOpts(configurePath string, repository integrity.Repository) (
@@ -250,7 +289,7 @@ func GetCliOpts(configurePath string, repository integrity.Repository) (
 			return nil, "", fmt.Errorf("failed to parse Tarantool CLI configuration: %s", err)
 		}
 
-		if err := mapstructure.Decode(rawConfigOpts, &cfg); err != nil {
+		if err := decodeConfig(rawConfigOpts, cfg); err != nil {
 			return nil, "", fmt.Errorf("failed to parse Tarantool CLI configuration: %s", err)
 		}
 

cli/configure/configure_test.go

@@ -334,6 +334,67 @@ func TestUpdateCliOpts(t *testing.T) {
 	assert.Equal(t, "./var/lib/vinyl", cliOpts.App.VinylDir)
 	assert.Equal(t, "./var/lib/snap", cliOpts.App.MemtxDir)
 	assert.Equal(t, filepath.Join(configDir, "..", "include_dir"), cliOpts.Env.IncludeDir)
-	assert.Equal(t, filepath.Join(configDir, ModulesPath), cliOpts.Modules.Directory)
+	assert.Equal(t, 1, len(cliOpts.Modules.Directories))
+	assert.Equal(t, filepath.Join(configDir, ModulesPath), cliOpts.Modules.Directories[0])
 	assert.Equal(t, configDir, cliOpts.Env.InstancesEnabled)
 }
+
+func TestGetCliOpts_modules_directory(t *testing.T) {
+	work_dir, err := os.Getwd()
+	require.NoError(t, err)
+	work_dir = filepath.Join(work_dir, "testdata/modules_cfg")
+
+	tests := []struct {
+		name        string
+		config      string
+		modules_dir config.FieldStringArrayType
+		cfg_path    string
+	}{
+		{
+			name:        "Single string relative path",
+			config:      "tt-modules1",
+			modules_dir: []string{filepath.Join(work_dir, "modules-dir")},
+			cfg_path:    "tt-modules1.yaml",
+		},
+		{
+			name:        "Single entry list",
+			config:      "tt-modules2",
+			modules_dir: []string{filepath.Join(work_dir, "modules-dir")},
+			cfg_path:    "tt-modules2.yml",
+		},
+		{
+			name:   "Multiple entries list",
+			config: "tt-modules3.",
+			modules_dir: []string{
+				filepath.Join(work_dir, "modules-dir"),
+				"/ext/path/modules",
+				filepath.Join(work_dir, "local_modules"),
+			},
+			cfg_path: "tt-modules3.yaml",
+		},
+		{
+			name:        "Empty list = default value",
+			config:      "tt-modules4.yaml",
+			modules_dir: []string{filepath.Join(work_dir, "modules")},
+			cfg_path:    "tt-modules4.yml",
+		},
+		{
+			name:        "Single string absolute path",
+			config:      "tt-modules5.yml",
+			modules_dir: []string{"/ext/path/modules"},
+			cfg_path:    "tt-modules5.yaml",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mockRepo := newMockRepository()
+			config := filepath.Join(work_dir, tt.config)
+			opts, cfg, err := GetCliOpts(config, &mockRepo)
+			require.NoError(t, err)
+			require.NotNil(t, opts.Modules)
+			require.Equal(t, tt.modules_dir, opts.Modules.Directories)
+			require.Equal(t, filepath.Join(work_dir, tt.cfg_path), cfg)
+		})
+	}
+}

cli/configure/testdata/modules_cfg/tt-modules1.yaml

@@ -0,0 +1,3 @@
+# Config with single string with relative path to modules directory.
+modules:
+  directory: modules-dir

cli/configure/testdata/modules_cfg/tt-modules2.yml

@@ -0,0 +1,4 @@
+# Config with single list entry of relative path to modules directory.
+modules:
+  directory:
+  - modules-dir

cli/configure/testdata/modules_cfg/tt-modules3.yaml

@@ -0,0 +1,6 @@
+# Multiple directory items.
+modules:
+  directory:
+    - modules-dir
+    - /ext/path/modules
+    - local_modules

cli/configure/testdata/modules_cfg/tt-modules4.yml

@@ -0,0 +1,3 @@
+# Empty config value, to use default.
+modules:
+  directory:

cli/configure/testdata/modules_cfg/tt-modules5.yaml

@@ -0,0 +1,3 @@
+# Config with single string with absolute path to modules directory.
+modules:
+  directory: /ext/path/modules

cli/init/init.go

@@ -168,11 +168,12 @@ func generateTtEnv(configPath string, sourceCfg configData) error {
 
 	directoriesToCreate := []string{
 		cfg.Env.InstancesEnabled,
-		cfg.Modules.Directory,
 		cfg.Env.IncludeDir,
 		cfg.Env.BinDir,
 		cfg.Repo.Install,
 	}
+	// FIXME: Need select only internal directories https://github.com/tarantool/tt/issues/1014
+	directoriesToCreate = append(directoriesToCreate, cfg.Modules.Directories...)
 	for _, templatesPathOpts := range cfg.Templates {
 		directoriesToCreate = append(directoriesToCreate, templatesPathOpts.Path)
 	}

cli/init/init_test.go

@@ -63,7 +63,7 @@ func checkDefaultEnv(t *testing.T, configName string, instancesEnabled string) {
 	assert.Equal(t, "var/run", cfg.App.RunDir)
 	assert.Equal(t, "var/log", cfg.App.LogDir)
 	assert.Equal(t, "bin", cfg.Env.BinDir)
-	assert.Equal(t, "modules", cfg.Modules.Directory)
+	assert.Equal(t, config.FieldStringArrayType{"modules"}, cfg.Modules.Directories)
 	assert.Equal(t, "distfiles", cfg.Repo.Install)
 	assert.Equal(t, "include", cfg.Env.IncludeDir)
 	assert.Equal(t, "templates", cfg.Templates[0].Path)

cli/init/templates/tt.yaml.default

@@ -1,6 +1,6 @@
 modules:
   # Directory where the external modules are stored.
-  directory: {{ .Modules.Directory }}
+  directory: {{ .Modules.Directories }}
 
 env:
   # Restart instance on failure.

cli/modules/modules.go

@@ -94,7 +94,8 @@ func getExternalModulesDir(cmdCtx *cmdcontext.CmdCtx, cliOpts *config.CliOpts) (
 	// 1. If a directory field is specified;
 	// 2. Specified path exists;
 	// 3. Path points to not a directory.
-	modulesDir := cliOpts.Modules.Directory
+	// FIXME: Add working with a list https://github.com/tarantool/tt/issues/1014
+	modulesDir := cliOpts.Modules.Directories[0]
 	if info, err := os.Stat(modulesDir); err == nil {
 		// TODO: Add warning in next patches, discussion
 		// what if the file exists, but access is denied, etc.

cli/pack/common.go

@@ -115,7 +115,7 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
 			cliOpts.Env.InstancesEnabled, cliOpts.Env.BinDir)
 	}
 	if cliOpts.Modules != nil {
-		envPaths = append(envPaths, cliOpts.Modules.Directory)
+		envPaths = append(envPaths, cliOpts.Modules.Directories...)
 	}
 	if cliOpts.Repo != nil {
 		envPaths = append(envPaths, cliOpts.Repo.Install)
@@ -204,24 +204,27 @@ func updateEnvPath(basePath string, packCtx *PackCtx, cliOpts *config.CliOpts) (
 // copyEnvModules copies tt modules.
 func copyEnvModules(bundleEnvPath string, packCtx *PackCtx, cliOpts, newOpts *config.CliOpts) {
 	if packCtx.WithoutModules || packCtx.CartridgeCompat || cliOpts.Modules == nil ||
-		cliOpts.Modules.Directory == "" {
+		len(cliOpts.Modules.Directories) == 0 {
 		return
 	}
 
-	if !util.IsDir(cliOpts.Modules.Directory) {
-		log.Debugf("Skip copying modules from %q: does not exist or not a directory",
-			cliOpts.Modules.Directory)
-	} else {
-		dir, err := os.Open(cliOpts.Modules.Directory)
-		if err != nil {
-			log.Warnf("cannot open %q for reading: %s", cliOpts.Modules.Directory, err)
-		}
-		if files, _ := dir.Readdir(1); len(files) == 0 {
-			return // No modules.
-		}
-		if err := copy.Copy(cliOpts.Modules.Directory,
-			util.JoinPaths(bundleEnvPath, newOpts.Modules.Directory)); err != nil {
-			log.Warnf("Failed to copy modules from %q: %s", cliOpts.Modules.Directory, err)
+	for _, directory := range cliOpts.Modules.Directories {
+		if !util.IsDir(directory) {
+			log.Debugf("Skip copying modules from %q: does not exist or not a directory",
+				directory)
+		} else {
+			dir, err := os.Open(directory)
+			if err != nil {
+				log.Warnf("cannot open %q for reading: %s", directory, err)
+			}
+			if files, _ := dir.Readdir(1); len(files) == 0 {
+				return // No modules.
+			}
+			// FIXME: Add working with a list https://github.com/tarantool/tt/issues/1014
+			if err := copy.Copy(directory,
+				util.JoinPaths(bundleEnvPath, newOpts.Modules.Directories[0])); err != nil {
+				log.Warnf("Failed to copy modules from %q: %s", directory, err)
+			}
 		}
 	}
 }

cli/pack/common_test.go

@@ -243,7 +243,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "var/run",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",
@@ -280,7 +280,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "var/run",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",
@@ -317,7 +317,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "/var/run/tarantool/bundle",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",
@@ -360,7 +360,7 @@ func Test_createNewOpts(t *testing.T) {
 					RunDir:   "var/run",
 				},
 				Modules: &config.ModulesOpts{
-					Directory: "modules",
+					Directories: []string{"modules"},
 				},
 				Repo: &config.RepoOpts{
 					Rocks:   "",