diff --git a/.gitignore b/.gitignore index 3297285..dcc8b0b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules +vendor bin releases ./leaps diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f639613..0000000 --- a/.gitmodules +++ /dev/null @@ -1,42 +0,0 @@ -[submodule "vendor/github.com/amir/raidman"] - path = vendor/github.com/amir/raidman - url = https://github.com/amir/raidman -[submodule "vendor/github.com/elazarl/go-bindata-assetfs"] - path = vendor/github.com/elazarl/go-bindata-assetfs - url = https://github.com/elazarl/go-bindata-assetfs -[submodule "vendor/github.com/garyburd/redigo"] - path = vendor/github.com/garyburd/redigo - url = https://github.com/garyburd/redigo -[submodule "vendor/github.com/golang/protobuf"] - path = vendor/github.com/golang/protobuf - url = https://github.com/golang/protobuf -[submodule "vendor/github.com/go-sql-driver/mysql"] - path = vendor/github.com/go-sql-driver/mysql - url = https://github.com/go-sql-driver/mysql -[submodule "vendor/github.com/jeffail/gabs"] - path = vendor/github.com/jeffail/gabs - url = https://github.com/jeffail/gabs -[submodule "vendor/github.com/jeffail/util"] - path = vendor/github.com/jeffail/util - url = https://github.com/jeffail/util -[submodule "vendor/github.com/lib/pq"] - path = vendor/github.com/lib/pq - url = https://github.com/lib/pq -[submodule "vendor/github.com/satori/go.uuid"] - path = vendor/github.com/satori/go.uuid - url = https://github.com/satori/go.uuid -[submodule "vendor/github.com/jteeuwen/go-bindata"] - path = vendor/github.com/jteeuwen/go-bindata - url = https://github.com/jteeuwen/go-bindata -[submodule "vendor/golang.org/x/net"] - path = vendor/golang.org/x/net - url = https://go.googlesource.com/net -[submodule "vendor/github.com/codemirror/codemirror"] - path = vendor/github.com/codemirror/codemirror - url = https://github.com/codemirror/codemirror -[submodule "vendor/github.com/vuejs/vue"] - path = vendor/github.com/vuejs/vue - url = https://github.com/vuejs/vue -[submodule "vendor/github.com/gorilla/websocket"] - path = vendor/github.com/gorilla/websocket - url = https://github.com/gorilla/websocket diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..b4ff20f --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,133 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/Azure/go-autorest" + packages = [ + "autorest", + "autorest/adal", + "autorest/azure", + "autorest/date" + ] + revision = "fc3b03a2d2d1f43fad3007038bd16f044f870722" + version = "v9.10.0" + +[[projects]] + name = "github.com/Jeffail/gabs" + packages = ["."] + revision = "2a3aa15961d5fee6047b8151b67ac2f08ba2c48c" + version = "1.0" + +[[projects]] + branch = "master" + name = "github.com/amir/raidman" + packages = [ + ".", + "proto" + ] + revision = "1ccc43bfb9c93cb401a4025e49c64ba71e5e668b" + +[[projects]] + name = "github.com/azure/azure-sdk-for-go" + packages = ["storage"] + revision = "21b68149ccf7c16b3f028bb4c7fd0ab458fe308f" + version = "v12.5.0-beta" + +[[projects]] + name = "github.com/cenkalti/backoff" + packages = ["."] + revision = "61153c768f31ee5f130071d08fc82b85208528de" + version = "v1.1.0" + +[[projects]] + name = "github.com/dgrijalva/jwt-go" + packages = ["."] + revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29" + version = "v3.1.0" + +[[projects]] + name = "github.com/elazarl/go-bindata-assetfs" + packages = ["."] + revision = "30f82fa23fd844bd5bb1e5f216db87fd77b5eb43" + version = "v1.0.0" + +[[projects]] + name = "github.com/garyburd/redigo" + packages = [ + "internal", + "redis" + ] + revision = "d1ed5c67e5794de818ea85e6b522fda02623a484" + version = "v1.5.0" + +[[projects]] + name = "github.com/go-sql-driver/mysql" + packages = ["."] + revision = "a0583e0143b1624142adab07e0e97fe106d99561" + version = "v1.3" + +[[projects]] + name = "github.com/golang/protobuf" + packages = ["proto"] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + name = "github.com/gorilla/websocket" + packages = ["."] + revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" + version = "v1.2.0" + +[[projects]] + branch = "master" + name = "github.com/kardianos/osext" + packages = ["."] + revision = "ae77be60afb1dcacde03767a8c37337fad28ac14" + +[[projects]] + branch = "master" + name = "github.com/lib/pq" + packages = [ + ".", + "oid" + ] + revision = "88edab0803230a3898347e77b474f8c1820a1f20" + +[[projects]] + name = "github.com/marstr/guid" + packages = ["."] + revision = "8bdf7d1a087ccc975cf37dd6507da50698fd19ca" + +[[projects]] + name = "github.com/satori/go.uuid" + packages = ["."] + revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" + version = "v1.2.0" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "proxy" + ] + revision = "cbe0f9307d0156177f9dd5dc85da1a31abc5f2fb" + +[[projects]] + name = "gopkg.in/alexcesaro/statsd.v2" + packages = ["."] + revision = "7fea3f0d2fab1ad973e641e51dba45443a311a90" + version = "v2.0.0" + +[[projects]] + branch = "v2" + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "d3ddb55d95c9c41adbf1b58c629f07f17442179c2a2927e3e38df7aa5fa99f6f" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..cd328f4 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,82 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/Jeffail/gabs" + version = "1.0.0" + +[[constraint]] + branch = "master" + name = "github.com/amir/raidman" + +[[constraint]] + name = "github.com/azure/azure-sdk-for-go" + version = "12.5.0-beta" + +[[constraint]] + name = "github.com/cenkalti/backoff" + version = "1.1.0" + +[[constraint]] + name = "github.com/elazarl/go-bindata-assetfs" + version = "1.0.0" + +[[constraint]] + name = "github.com/garyburd/redigo" + version = "1.5.0" + +[[constraint]] + name = "github.com/go-sql-driver/mysql" + version = "1.3.0" + +[[constraint]] + name = "github.com/gorilla/websocket" + version = "1.2.0" + +[[constraint]] + branch = "master" + name = "github.com/kardianos/osext" + +[[constraint]] + branch = "master" + name = "github.com/lib/pq" + +[[constraint]] + name = "github.com/satori/go.uuid" + version = "1.2.0" + +[[constraint]] + name = "gopkg.in/alexcesaro/statsd.v2" + version = "2.0.0" + +[[constraint]] + branch = "v2" + name = "gopkg.in/yaml.v2" + +[prune] + go-tests = true + unused-packages = true diff --git a/README.md b/README.md index 45654d7..9b11638 100644 --- a/README.md +++ b/README.md @@ -58,55 +58,38 @@ for your OS from [the latest releases page][3]. ### From homebrew -```bash +``` sh brew install leaps leaps -h ``` ### Build with Go -```bash -go get github.com/jeffail/leaps/cmd/... +``` sh +go get github.com/Jeffail/leaps/cmd/... leaps -h ``` ## Vendoring -Versions of go above 1.6 should automatically `go get` all vendored libraries. -Otherwise, while cloning use `--recursive`: +Leaps uses [dep][https://github.com/golang/dep] for managing its dependencies. +To obtain the correct versions of all dependencies use: -`git clone https://github.com/jeffail/leaps --recursive` - -Or, if the repo is already cloned, get the latest libraries with: - -`git submodule update --init` - -To add new libraries simply run: - -``` -PACKAGE=github.com/jeffail/util -git submodule add https://$PACKAGE vendor/$PACKAGE" -``` - -It might be handy to set yourself a function for this in your `.bashrc`: - -```bash -function go-add-vendor { - git submodule add https://$1 vendor/$1 -} +``` sh +dep ensure ``` ## System compatibility -OS | Status ----------------- | ------ -OSX x86_64 | Supported, tested -Linux x86 | Supported -Linux x86_64 | Supported, tested -Linux ARMv5 | Builds -Linux ARMv7 | Supported, tested -Windows x86 | Builds -Windows x86_64 | Builds +OS | Status +------------------ | ------ +OSX `x86_64` | Supported, tested +Linux `x86` | Supported +Linux `x86_64` | Supported, tested +Linux `ARMv5` | Builds +Linux `ARMv7` | Supported, tested +Windows `x86` | Builds +Windows `x86_64` | Builds ## Contributing and customizing @@ -122,7 +105,7 @@ Ashley Jeffs [1]: https://godoc.org/github.com/Jeffail/leaps [2]: client/javascript/README.md [3]: https://github.com/Jeffail/leaps/releases/latest -[4]: https://godoc.org/github.com/jeffail/leaps?status.svg -[5]: http://godoc.org/github.com/jeffail/leaps -[6]: https://goreportcard.com/badge/github.com/jeffail/leaps +[4]: https://godoc.org/github.com/Jeffail/leaps?status.svg +[5]: http://godoc.org/github.com/Jeffail/leaps +[6]: https://goreportcard.com/badge/github.com/Jeffail/leaps [7]: https://goreportcard.com/report/jeffail/leaps diff --git a/cmd/leaps/leaps.go b/cmd/leaps/leaps.go index 7913047..63ca763 100644 --- a/cmd/leaps/leaps.go +++ b/cmd/leaps/leaps.go @@ -38,16 +38,16 @@ import ( "syscall" "time" + "github.com/Jeffail/leaps/lib/acl" + "github.com/Jeffail/leaps/lib/api" + apiio "github.com/Jeffail/leaps/lib/api/io" + "github.com/Jeffail/leaps/lib/audit" + "github.com/Jeffail/leaps/lib/curator" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/util" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" "github.com/gorilla/websocket" - "github.com/jeffail/leaps/lib/acl" - "github.com/jeffail/leaps/lib/api" - apiio "github.com/jeffail/leaps/lib/api/io" - "github.com/jeffail/leaps/lib/audit" - "github.com/jeffail/leaps/lib/curator" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/util" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" ) //------------------------------------------------------------------------------ @@ -201,9 +201,10 @@ If a path is not specified the current directory is shared instead. logger := log.NewLogger(os.Stdout, logConf) statConf := metrics.NewConfig() - statConf.Prefix = "leaps" + statConf.Type = "http" + statConf.HTTP.Prefix = "leaps" - stats, err := metrics.New(statConf) + stats, err := metrics.NewHTTP(statConf) if err != nil { fmt.Fprintln(os.Stderr, fmt.Sprintf("Metrics init error: %v\n", err)) return @@ -323,7 +324,9 @@ If a path is not specified the current directory is shared instead. w.Write(data) }) - handle("/stats", "Lists all aggregated metrics as a json blob.", stats.JSONHandler()) + if hStats, ok := stats.(*metrics.HTTP); ok { + handle("/stats", "Lists all aggregated metrics as a json blob.", hStats.JSONHandler()) + } wwwPath := gopath.Join("/", subdirPath) stripPath := "" diff --git a/lib/acl/file_exists.go b/lib/acl/file_exists.go index 7b94735..aad24f8 100644 --- a/lib/acl/file_exists.go +++ b/lib/acl/file_exists.go @@ -31,7 +31,7 @@ import ( "sync" "time" - "github.com/jeffail/util/log" + "github.com/Jeffail/leaps/lib/util/service/log" ) //-------------------------------------------------------------------------------------------------- diff --git a/lib/acl/file_exists_test.go b/lib/acl/file_exists_test.go index 6fef8db..6910990 100644 --- a/lib/acl/file_exists_test.go +++ b/lib/acl/file_exists_test.go @@ -26,7 +26,7 @@ import ( "os" "testing" - "github.com/jeffail/util/log" + "github.com/Jeffail/leaps/lib/util/service/log" ) //-------------------------------------------------------------------------------------------------- diff --git a/lib/acl/redis_based.go b/lib/acl/redis_based.go index 7efc866..840ab2c 100644 --- a/lib/acl/redis_based.go +++ b/lib/acl/redis_based.go @@ -29,7 +29,7 @@ import ( "time" "github.com/garyburd/redigo/redis" - "github.com/jeffail/util/log" + "github.com/Jeffail/leaps/lib/util/service/log" ) //-------------------------------------------------------------------------------------------------- diff --git a/lib/api/cmd_broker.go b/lib/api/cmd_broker.go index 2f48583..27c37b6 100644 --- a/lib/api/cmd_broker.go +++ b/lib/api/cmd_broker.go @@ -27,9 +27,9 @@ import ( "sync" "time" - "github.com/jeffail/leaps/lib/api/events" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) //------------------------------------------------------------------------------ @@ -57,7 +57,7 @@ type CMDBroker struct { timeout time.Duration logger log.Modular - stats metrics.Aggregator + stats metrics.Type emittersMut sync.Mutex closeChan chan struct{} @@ -69,7 +69,7 @@ func NewCMDBroker( cmdRunner CMDRunner, timeout time.Duration, logger log.Modular, - stats metrics.Aggregator, + stats metrics.Type, ) *CMDBroker { b := &CMDBroker{ cmds: cmds, diff --git a/lib/api/cmd_broker_test.go b/lib/api/cmd_broker_test.go index fa479e2..63af88f 100644 --- a/lib/api/cmd_broker_test.go +++ b/lib/api/cmd_broker_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/api/events" ) //------------------------------------------------------------------------------ diff --git a/lib/api/common_test.go b/lib/api/common_test.go index 1f520a7..0c006fb 100644 --- a/lib/api/common_test.go +++ b/lib/api/common_test.go @@ -29,17 +29,17 @@ import ( "reflect" "time" - "github.com/jeffail/leaps/lib/api/events" - "github.com/jeffail/leaps/lib/binder" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/binder" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) //------------------------------------------------------------------------------ -var logger, stats = func() (log.Modular, metrics.Aggregator) { +var logger, stats = func() (log.Modular, metrics.Type) { logConf := log.NewLoggerConfig() logConf.LogLevel = "OFF" return log.NewLogger(os.Stdout, logConf), metrics.DudType{} diff --git a/lib/api/curator_session.go b/lib/api/curator_session.go index 6156e3a..404f1ce 100644 --- a/lib/api/curator_session.go +++ b/lib/api/curator_session.go @@ -28,12 +28,12 @@ import ( "sync" "time" - "github.com/jeffail/leaps/lib/api/events" - "github.com/jeffail/leaps/lib/binder" - "github.com/jeffail/leaps/lib/curator" - "github.com/jeffail/leaps/lib/text" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/binder" + "github.com/Jeffail/leaps/lib/curator" + "github.com/Jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) //------------------------------------------------------------------------------ @@ -47,7 +47,7 @@ type CuratorSession struct { timeout time.Duration logger log.Modular - stats metrics.Aggregator + stats metrics.Type username string uuid string @@ -64,7 +64,7 @@ func NewCuratorSession( cur curator.Type, timeout time.Duration, logger log.Modular, - stats metrics.Aggregator, + stats metrics.Type, ) *CuratorSession { s := &CuratorSession{ username: username, diff --git a/lib/api/curator_session_test.go b/lib/api/curator_session_test.go index e1242e1..ecefa1f 100644 --- a/lib/api/curator_session_test.go +++ b/lib/api/curator_session_test.go @@ -27,9 +27,9 @@ import ( "testing" "time" - "github.com/jeffail/leaps/lib/api/events" - "github.com/jeffail/leaps/lib/binder" - "github.com/jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/binder" + "github.com/Jeffail/leaps/lib/text" ) //------------------------------------------------------------------------------ diff --git a/lib/api/emitter.go b/lib/api/emitter.go index 5fea176..e6cbda2 100644 --- a/lib/api/emitter.go +++ b/lib/api/emitter.go @@ -22,7 +22,7 @@ THE SOFTWARE. package api -import "github.com/jeffail/leaps/lib/api/events" +import "github.com/Jeffail/leaps/lib/api/events" //------------------------------------------------------------------------------ diff --git a/lib/api/events/events.go b/lib/api/events/events.go index caf559b..c89ef0d 100644 --- a/lib/api/events/events.go +++ b/lib/api/events/events.go @@ -23,7 +23,7 @@ THE SOFTWARE. package events import ( - "github.com/jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/text" ) //------------------------------------------------------------------------------ diff --git a/lib/api/global_metadata_broker.go b/lib/api/global_metadata_broker.go index be4c4ec..f841e5a 100644 --- a/lib/api/global_metadata_broker.go +++ b/lib/api/global_metadata_broker.go @@ -27,9 +27,9 @@ import ( "sync" "time" - "github.com/jeffail/leaps/lib/api/events" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) //------------------------------------------------------------------------------ @@ -44,7 +44,7 @@ type GlobalMetadataBroker struct { timeout time.Duration logger log.Modular - stats metrics.Aggregator + stats metrics.Type userMapMut sync.Mutex emittersMut sync.Mutex @@ -54,7 +54,7 @@ type GlobalMetadataBroker struct { func NewGlobalMetadataBroker( timeout time.Duration, logger log.Modular, - stats metrics.Aggregator, + stats metrics.Type, ) *GlobalMetadataBroker { return &GlobalMetadataBroker{ userMap: make(map[string]events.UserSubscriptions), diff --git a/lib/api/global_metadata_broker_test.go b/lib/api/global_metadata_broker_test.go index 2d6f9ac..593b98f 100644 --- a/lib/api/global_metadata_broker_test.go +++ b/lib/api/global_metadata_broker_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/api/events" ) //------------------------------------------------------------------------------ diff --git a/lib/api/io/json_emitter.go b/lib/api/io/json_emitter.go index 6e7933e..563c8f7 100644 --- a/lib/api/io/json_emitter.go +++ b/lib/api/io/json_emitter.go @@ -25,8 +25,8 @@ package io import ( "encoding/json" - "github.com/jeffail/leaps/lib/api" - "github.com/jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/api" + "github.com/Jeffail/leaps/lib/api/events" ) //------------------------------------------------------------------------------ diff --git a/lib/api/io/json_emitter_test.go b/lib/api/io/json_emitter_test.go index 8bd2431..dc7ac80 100644 --- a/lib/api/io/json_emitter_test.go +++ b/lib/api/io/json_emitter_test.go @@ -29,7 +29,7 @@ import ( "testing" "time" - "github.com/jeffail/leaps/lib/api/events" + "github.com/Jeffail/leaps/lib/api/events" ) //------------------------------------------------------------------------------ diff --git a/lib/audit/interface.go b/lib/audit/interface.go index ab97985..4f9d8ba 100644 --- a/lib/audit/interface.go +++ b/lib/audit/interface.go @@ -21,7 +21,7 @@ package audit import ( - "github.com/jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/text" ) //------------------------------------------------------------------------------ diff --git a/lib/audit/to_json.go b/lib/audit/to_json.go index 5d0457f..d9668aa 100644 --- a/lib/audit/to_json.go +++ b/lib/audit/to_json.go @@ -25,8 +25,8 @@ import ( "os" "sync" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" ) //------------------------------------------------------------------------------ diff --git a/lib/audit/to_json_test.go b/lib/audit/to_json_test.go index 9e3fcd5..919600b 100644 --- a/lib/audit/to_json_test.go +++ b/lib/audit/to_json_test.go @@ -26,8 +26,8 @@ import ( "testing" "time" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" ) //------------------------------------------------------------------------------ diff --git a/lib/binder/binder.go b/lib/binder/binder.go index 76faa4f..912f767 100644 --- a/lib/binder/binder.go +++ b/lib/binder/binder.go @@ -27,11 +27,11 @@ import ( "sync" "time" - "github.com/jeffail/leaps/lib/audit" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/audit" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) //------------------------------------------------------------------------------ @@ -69,7 +69,7 @@ type impl struct { auditor audit.Auditor log log.Modular - stats metrics.Aggregator + stats metrics.Type // Clients clients []*binderClient @@ -92,7 +92,7 @@ func New( config Config, errorChan chan<- Error, log log.Modular, - stats metrics.Aggregator, + stats metrics.Type, auditor audit.Auditor, ) (Type, error) { binder := impl{ diff --git a/lib/binder/binder_test.go b/lib/binder/binder_test.go index c5bec8c..0f5e0e6 100644 --- a/lib/binder/binder_test.go +++ b/lib/binder/binder_test.go @@ -31,9 +31,9 @@ import ( "testing" "time" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" - "github.com/jeffail/leaps/lib/util" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/util" ) //-------------------------------------------------------------------------------------------------- diff --git a/lib/binder/common_test.go b/lib/binder/common_test.go index ceb37ca..631d8a5 100644 --- a/lib/binder/common_test.go +++ b/lib/binder/common_test.go @@ -3,11 +3,11 @@ package binder import ( "os" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) -func loggerAndStats() (log.Modular, metrics.Aggregator) { +func loggerAndStats() (log.Modular, metrics.Type) { logConf := log.NewLoggerConfig() logConf.LogLevel = "OFF" diff --git a/lib/binder/interface.go b/lib/binder/interface.go index c4cdb23..ea02258 100644 --- a/lib/binder/interface.go +++ b/lib/binder/interface.go @@ -25,8 +25,8 @@ package binder import ( "time" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" ) //------------------------------------------------------------------------------ diff --git a/lib/binder/portal.go b/lib/binder/portal.go index cc67598..9f45b1f 100644 --- a/lib/binder/portal.go +++ b/lib/binder/portal.go @@ -26,8 +26,8 @@ import ( "errors" "time" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" ) //------------------------------------------------------------------------------ diff --git a/lib/binder/portal_types.go b/lib/binder/portal_types.go index dadb4dc..bf4ae2a 100644 --- a/lib/binder/portal_types.go +++ b/lib/binder/portal_types.go @@ -22,7 +22,7 @@ THE SOFTWARE. package binder -import "github.com/jeffail/leaps/lib/text" +import "github.com/Jeffail/leaps/lib/text" //------------------------------------------------------------------------------ diff --git a/lib/curator/curator.go b/lib/curator/curator.go index 32b215e..90a922b 100644 --- a/lib/curator/curator.go +++ b/lib/curator/curator.go @@ -28,12 +28,12 @@ import ( "sync" "time" - "github.com/jeffail/leaps/lib/acl" - "github.com/jeffail/leaps/lib/audit" - "github.com/jeffail/leaps/lib/binder" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/acl" + "github.com/Jeffail/leaps/lib/audit" + "github.com/Jeffail/leaps/lib/binder" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) //------------------------------------------------------------------------------ @@ -67,7 +67,7 @@ type Impl struct { auditors AuditorContainer log log.Modular - stats metrics.Aggregator + stats metrics.Type // Binders openBinders map[string]binder.Type @@ -83,7 +83,7 @@ type Impl struct { func New( config Config, log log.Modular, - stats metrics.Aggregator, + stats metrics.Type, auth acl.Authenticator, store store.Type, auditors AuditorContainer, diff --git a/lib/curator/curator_test.go b/lib/curator/curator_test.go index 6207f46..d17c492 100644 --- a/lib/curator/curator_test.go +++ b/lib/curator/curator_test.go @@ -30,15 +30,15 @@ import ( "testing" "time" - "github.com/jeffail/leaps/lib/acl" - "github.com/jeffail/leaps/lib/binder" - "github.com/jeffail/leaps/lib/store" - "github.com/jeffail/leaps/lib/text" - "github.com/jeffail/util/log" - "github.com/jeffail/util/metrics" + "github.com/Jeffail/leaps/lib/acl" + "github.com/Jeffail/leaps/lib/binder" + "github.com/Jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/text" + "github.com/Jeffail/leaps/lib/util/service/log" + "github.com/Jeffail/leaps/lib/util/service/metrics" ) -func loggerAndStats() (log.Modular, metrics.Aggregator) { +func loggerAndStats() (log.Modular, metrics.Type) { logConf := log.NewLoggerConfig() logConf.LogLevel = "OFF" @@ -48,7 +48,7 @@ func loggerAndStats() (log.Modular, metrics.Aggregator) { return logger, stats } -func authAndStore(logger log.Modular, stats metrics.Aggregator) (acl.Authenticator, store.Type) { +func authAndStore(logger log.Modular, stats metrics.Type) (acl.Authenticator, store.Type) { return &acl.Anarchy{AllowCreate: true}, store.NewMemory() } diff --git a/lib/curator/interface.go b/lib/curator/interface.go index 8349be6..d3b6449 100644 --- a/lib/curator/interface.go +++ b/lib/curator/interface.go @@ -25,9 +25,9 @@ package curator import ( "time" - "github.com/jeffail/leaps/lib/audit" - "github.com/jeffail/leaps/lib/binder" - "github.com/jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/audit" + "github.com/Jeffail/leaps/lib/binder" + "github.com/Jeffail/leaps/lib/store" ) //------------------------------------------------------------------------------ diff --git a/lib/store/document.go b/lib/store/document.go index 842c567..0f28889 100644 --- a/lib/store/document.go +++ b/lib/store/document.go @@ -22,7 +22,7 @@ THE SOFTWARE. package store -import "github.com/jeffail/leaps/lib/util" +import "github.com/Jeffail/leaps/lib/util" //------------------------------------------------------------------------------ diff --git a/lib/text/ot_buffer_test.go b/lib/text/ot_buffer_test.go index 2a2c55f..0fcbb41 100644 --- a/lib/text/ot_buffer_test.go +++ b/lib/text/ot_buffer_test.go @@ -28,7 +28,7 @@ import ( "io/ioutil" "testing" - "github.com/jeffail/leaps/lib/store" + "github.com/Jeffail/leaps/lib/store" ) func TestTextOTBufferSimpleTransforms(t *testing.T) { diff --git a/lib/util/service/bootstrap.go b/lib/util/service/bootstrap.go new file mode 100644 index 0000000..09202cb --- /dev/null +++ b/lib/util/service/bootstrap.go @@ -0,0 +1,161 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* +Package util provides a few bootstrapping utilities for golang services, and a consistent API to +wrap third parties libraries for things such as stats aggregation and logging. +*/ +package service + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" +) + +/*-------------------------------------------------------------------------------------------------- + */ + +var ( + version string + dateBuilt string + showVersion *bool + showConfigJSON *bool + showConfigYAML *bool + configPath *string +) + +func init() { + showVersion = flag.Bool("version", false, "Display version info, then exit") + showConfigJSON = flag.Bool("print-json", false, "Print loaded configuration as JSON, then exit") + showConfigYAML = flag.Bool("print-yaml", false, "Print loaded configuration as YAML, then exit") + configPath = flag.String("c", "", "Path to a configuration file") +} + +/*-------------------------------------------------------------------------------------------------- + */ + +func readConfig(path string, config interface{}) error { + configBytes, err := ioutil.ReadFile(path) + if err != nil { + return err + } + ext := filepath.Ext(path) + if ".js" == ext || ".json" == ext { + if err = json.Unmarshal(configBytes, config); err != nil { + return err + } + } else if ".yml" == ext || ".yaml" == ext { + if err = yaml.Unmarshal(configBytes, config); err != nil { + return err + } + } else { + return fmt.Errorf("config file extension not recognised: %v", path) + } + return nil +} + +/* +Bootstrap - bootstraps the configuration loading, parsing and reporting for a service through cmd +flags. The argument configPtr should be a pointer to a serializable configuration object with all +default values. + +configPtr - should be a pointer to a config struct, which contains default values and should be +populated with a users config values if applicable. For an example look at the stats and logger +files. + +defaultConfigPaths - if there are known standard configuration paths then you can list them here, +if the user neglects to specify a config then bootstrap will iterate these paths and read the first +one that exists, if any. + +Bootstrap allows a user to do the following: +- Print version and build info and exit +- Load an optional configuration file (supports JSON, YAML) +- Print the config file (supports JSON, YAML) and exit + +NOTE: The user may request a version and build time stamp, in which case Bootstrap will print the +values of util.Version and util.DateBuilt. To populate those values you must run go build with the +following: + +-ldflags "-X github.com/jeffail/util.version $(VERSION) \ + -X github.com/jeffail/util.dateBuilt $(DATE)" + +Returns a flag indicating whether the service should continue or not. +*/ +func Bootstrap(configPtr interface{}, defaultConfigPaths ...string) bool { + // Ensure that cmd flags are parsed. + if !flag.Parsed() { + flag.Parse() + } + + // If the user wants the version we print it. + if *showVersion { + fmt.Printf("Version: %v\nDate: %v\n", version, dateBuilt) + return false + } + + if len(*configPath) > 0 { + if err := readConfig(*configPath, configPtr); err != nil { + fmt.Fprintf(os.Stderr, "Configuration file read error: %v\n", err) + return false + } + } else { + // Iterate default config paths + for _, path := range defaultConfigPaths { + if _, err := os.Stat(path); err == nil { + fmt.Fprintf(os.Stderr, "Config file not specified, reading from %v\n", path) + + if err = readConfig(path, configPtr); err != nil { + fmt.Fprintf(os.Stderr, "Configuration file read error: %v\n", err) + return false + } + break + } + } + } + + // If the user wants the configuration to be printed we do so and then exit. + if *showConfigJSON { + if configJSON, err := json.MarshalIndent(configPtr, "", "\t"); err == nil { + fmt.Println(string(configJSON)) + } else { + fmt.Fprintln(os.Stderr, fmt.Sprintf("Configuration marshal error: %v", err)) + } + return false + } else if *showConfigYAML { + if configYAML, err := yaml.Marshal(configPtr); err == nil { + fmt.Println(string(configYAML)) + } else { + fmt.Fprintln(os.Stderr, fmt.Sprintf("Configuration marshal error: %v", err)) + } + return false + } + return true +} + +/*-------------------------------------------------------------------------------------------------- + */ diff --git a/lib/util/service/log/interface.go b/lib/util/service/log/interface.go new file mode 100644 index 0000000..9d33071 --- /dev/null +++ b/lib/util/service/log/interface.go @@ -0,0 +1,50 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package log + +/*-------------------------------------------------------------------------------------------------- + */ + +// Modular - A log printer that allows you to branch new modules. +type Modular interface { + NewModule(prefix string) Modular + + Fatalf(message string, other ...interface{}) + Errorf(message string, other ...interface{}) + Warnf(message string, other ...interface{}) + Infof(message string, other ...interface{}) + Debugf(message string, other ...interface{}) + Tracef(message string, other ...interface{}) + + Fatalln(message string) + Errorln(message string) + Warnln(message string) + Infoln(message string) + Debugln(message string) + Traceln(message string) + + Output(calldepth int, s string) error +} + +/*-------------------------------------------------------------------------------------------------- + */ diff --git a/lib/util/service/log/logger.go b/lib/util/service/log/logger.go new file mode 100644 index 0000000..bceac94 --- /dev/null +++ b/lib/util/service/log/logger.go @@ -0,0 +1,302 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package log + +import ( + "fmt" + "io" + "strconv" + "strings" + "time" +) + +//-------------------------------------------------------------------------------------------------- + +// Logger level constants +const ( + LogOff int = 0 + LogFatal int = 1 + LogError int = 2 + LogWarn int = 3 + LogInfo int = 4 + LogDebug int = 5 + LogTrace int = 6 + LogAll int = 7 +) + +// intToLogLevel - Converts an integer into a human readable log level. +func intToLogLevel(i int) string { + switch i { + case LogOff: + return "OFF" + case LogFatal: + return "FATAL" + case LogError: + return "ERROR" + case LogWarn: + return "WARN" + case LogInfo: + return "INFO" + case LogDebug: + return "DEBUG" + case LogTrace: + return "TRACE" + case LogAll: + return "ALL" + } + return "ALL" +} + +// logLevelToInt - Converts a human readable log level into an integer value. +func logLevelToInt(level string) int { + levelUpper := strings.ToUpper(level) + switch levelUpper { + case "OFF": + return LogOff + case "FATAL": + return LogFatal + case "ERROR": + return LogError + case "WARN": + return LogWarn + case "INFO": + return LogInfo + case "DEBUG": + return LogDebug + case "TRACE": + return LogTrace + case "ALL": + return LogAll + } + return -1 +} + +//-------------------------------------------------------------------------------------------------- + +// LoggerConfig - Holds configuration options for a logger object. +type LoggerConfig struct { + Prefix string `json:"prefix" yaml:"prefix"` + LogLevel string `json:"log_level" yaml:"log_level"` + AddTimeStamp bool `json:"add_timestamp" yaml:"add_timestamp"` + JSONFormat bool `json:"json_format" yaml:"json_format"` +} + +// NewLoggerConfig - Returns a logger configuration with the default values for each field. +func NewLoggerConfig() LoggerConfig { + return LoggerConfig{ + Prefix: "service", + LogLevel: "INFO", + AddTimeStamp: true, + JSONFormat: false, + } +} + +//-------------------------------------------------------------------------------------------------- + +// Logger - A logger object with support for levelled logging and modular components. +type Logger struct { + stream io.Writer + config LoggerConfig + level int +} + +// NewLogger - Create and return a new logger object. +func NewLogger(stream io.Writer, config LoggerConfig) Modular { + logger := Logger{ + stream: stream, + config: config, + level: logLevelToInt(config.LogLevel), + } + return &logger +} + +// NewModule - Creates a new logger object from the previous, using the same configuration, but adds +// an extra prefix to represent a submodule. +func (l *Logger) NewModule(prefix string) Modular { + config := l.config + config.Prefix = fmt.Sprintf("%v%v", config.Prefix, prefix) + + return &Logger{ + stream: l.stream, + config: config, + level: l.level, + } +} + +//-------------------------------------------------------------------------------------------------- + +// printf - Prints a log message with any configured extras prepended. +func (l *Logger) printf(message, level string, other ...interface{}) { + if l.config.JSONFormat { + if l.config.AddTimeStamp { + fmt.Fprintf(l.stream, fmt.Sprintf( + "{\"timestamp\":\"%v\",\"level\":\"%v\",\"service\":\"%v\",\"message\":%v}\n", + time.Now().Format(time.RFC3339), level, l.config.Prefix, + strconv.QuoteToASCII(message), + ), other...) + } else { + fmt.Fprintf(l.stream, fmt.Sprintf( + "{\"level\":\"%v\",\"service\":\"%v\",\"message\":%v}\n", + level, l.config.Prefix, + strconv.QuoteToASCII(message), + ), other...) + } + } else { + if l.config.AddTimeStamp { + fmt.Fprintf(l.stream, fmt.Sprintf( + "%v | %v | %v | %v", + time.Now().Format(time.RFC3339), level, l.config.Prefix, message, + ), other...) + } else { + fmt.Fprintf(l.stream, fmt.Sprintf( + "%v | %v | %v", level, l.config.Prefix, message, + ), other...) + } + } +} + +// printLine - Prints a log message with any configured extras prepended. +func (l *Logger) printLine(message, level string) { + if l.config.JSONFormat { + if l.config.AddTimeStamp { + fmt.Fprintf(l.stream, + "{\"timestamp\":\"%v\",\"level\":\"%v\",\"service\":\"%v\",\"message\":%v}\n", + time.Now().Format(time.RFC3339), level, l.config.Prefix, + strconv.QuoteToASCII(message), + ) + } else { + fmt.Fprintf(l.stream, + "{\"level\":\"%v\",\"service\":\"%v\",\"message\":%v}\n", + level, l.config.Prefix, + strconv.QuoteToASCII(message), + ) + } + } else { + if l.config.AddTimeStamp { + fmt.Fprintf( + l.stream, "%v | %v | %v | %v\n", + time.Now().Format(time.RFC3339), level, l.config.Prefix, message, + ) + } else { + fmt.Fprintf(l.stream, "%v | %v | %v\n", level, l.config.Prefix, message) + } + } +} + +//-------------------------------------------------------------------------------------------------- + +// Fatalf - Print a fatal message to the console. Does NOT cause panic. +func (l *Logger) Fatalf(message string, other ...interface{}) { + if LogFatal <= l.level { + l.printf(message, "FATAL", other...) + } +} + +// Errorf - Print an error message to the console. +func (l *Logger) Errorf(message string, other ...interface{}) { + if LogError <= l.level { + l.printf(message, "ERROR", other...) + } +} + +// Warnf - Print a warning message to the console. +func (l *Logger) Warnf(message string, other ...interface{}) { + if LogWarn <= l.level { + l.printf(message, "WARN", other...) + } +} + +// Infof - Print an information message to the console. +func (l *Logger) Infof(message string, other ...interface{}) { + if LogInfo <= l.level { + l.printf(message, "INFO", other...) + } +} + +// Debugf - Print a debug message to the console. +func (l *Logger) Debugf(message string, other ...interface{}) { + if LogDebug <= l.level { + l.printf(message, "DEBUG", other...) + } +} + +// Tracef - Print a trace message to the console. +func (l *Logger) Tracef(message string, other ...interface{}) { + if LogTrace <= l.level { + l.printf(message, "TRACE", other...) + } +} + +//-------------------------------------------------------------------------------------------------- + +// Fatalln - Print a fatal message to the console. Does NOT cause panic. +func (l *Logger) Fatalln(message string) { + if LogFatal <= l.level { + l.printLine(message, "FATAL") + } +} + +// Errorln - Print an error message to the console. +func (l *Logger) Errorln(message string) { + if LogError <= l.level { + l.printLine(message, "ERROR") + } +} + +// Warnln - Print a warning message to the console. +func (l *Logger) Warnln(message string) { + if LogWarn <= l.level { + l.printLine(message, "WARN") + } +} + +// Infoln - Print an information message to the console. +func (l *Logger) Infoln(message string) { + if LogInfo <= l.level { + l.printLine(message, "INFO") + } +} + +// Debugln - Print a debug message to the console. +func (l *Logger) Debugln(message string) { + if LogDebug <= l.level { + l.printLine(message, "DEBUG") + } +} + +// Traceln - Print a trace message to the console. +func (l *Logger) Traceln(message string) { + if LogTrace <= l.level { + l.printLine(message, "TRACE") + } +} + +//-------------------------------------------------------------------------------------------------- + +// Output - Prints s to our output. Calldepth is ignored. +func (l *Logger) Output(calldepth int, s string) error { + io.WriteString(l.stream, s) + return nil +} + +//-------------------------------------------------------------------------------------------------- diff --git a/lib/util/service/log/logger_test.go b/lib/util/service/log/logger_test.go new file mode 100644 index 0000000..b48d1d5 --- /dev/null +++ b/lib/util/service/log/logger_test.go @@ -0,0 +1,143 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package log + +import ( + "fmt" + "testing" +) + +type LogBuffer struct { + data string +} + +func (l *LogBuffer) Write(p []byte) (n int, err error) { + l.data = fmt.Sprintf("%v%v", l.data, string(p)) + return len(p), nil +} + +func TestModules(t *testing.T) { + loggerConfig := NewLoggerConfig() + loggerConfig.AddTimeStamp = false + loggerConfig.Prefix = "root" + loggerConfig.LogLevel = "WARN" + + buf := LogBuffer{data: ""} + + logger := NewLogger(&buf, loggerConfig) + logger.Warnln("Warning message root module") + + logger2 := logger.NewModule(".foo") + logger2.Warnln("Warning message root.foo module") + + logger3 := logger.NewModule(".foo2") + logger3.Warnln("Warning message root.foo2 module") + + logger4 := logger2.NewModule(".bar") + logger4.Warnln("Warning message root.foo.bar module") + + expected := "WARN | root | Warning message root module\n" + + "WARN | root.foo | Warning message root.foo module\n" + + "WARN | root.foo2 | Warning message root.foo2 module\n" + + "WARN | root.foo.bar | Warning message root.foo.bar module\n" + + if expected != buf.data { + t.Errorf("%v != %v", expected, buf.data) + } +} + +func TestFormattedLogging(t *testing.T) { + loggerConfig := NewLoggerConfig() + loggerConfig.AddTimeStamp = false + loggerConfig.Prefix = "test" + loggerConfig.LogLevel = "WARN" + + buf := LogBuffer{data: ""} + + logger := NewLogger(&buf, loggerConfig) + logger.Fatalf("fatal test %v\n", 1) + logger.Errorf("error test %v\n", 2) + logger.Warnf("warn test %v\n", 3) + logger.Infof("info test %v\n", 4) + logger.Debugf("info test %v\n", 5) + logger.Tracef("trace test %v\n", 6) + + expected := "FATAL | test | fatal test 1\nERROR | test | error test 2\nWARN | test | warn test 3\n" + + if expected != buf.data { + t.Errorf("%v != %v", expected, buf.data) + } +} + +func TestLineLogging(t *testing.T) { + loggerConfig := NewLoggerConfig() + loggerConfig.AddTimeStamp = false + loggerConfig.Prefix = "test" + loggerConfig.LogLevel = "WARN" + + buf := LogBuffer{data: ""} + + logger := NewLogger(&buf, loggerConfig) + logger.Fatalln("fatal test") + logger.Errorln("error test") + logger.Warnln("warn test") + logger.Infoln("info test") + logger.Debugln("info test") + logger.Traceln("trace test") + + expected := "FATAL | test | fatal test\nERROR | test | error test\nWARN | test | warn test\n" + + if expected != buf.data { + t.Errorf("%v != %v", expected, buf.data) + } +} + +type LogCounter struct { + count int +} + +func (l *LogCounter) Write(p []byte) (n int, err error) { + l.count++ + return len(p), nil +} + +func TestLogLevels(t *testing.T) { + for i := 0; i < LogAll; i++ { + loggerConfig := NewLoggerConfig() + loggerConfig.LogLevel = intToLogLevel(i) + + buf := LogCounter{count: 0} + + logger := NewLogger(&buf, loggerConfig) + logger.Fatalln("fatal test") + logger.Errorln("error test") + logger.Warnln("warn test") + logger.Infoln("info test") + logger.Debugln("info test") + logger.Traceln("trace test") + + if i != buf.count { + t.Errorf("Wrong log count for [%v], %v != %v", loggerConfig.LogLevel, i, buf.count) + } + } +} diff --git a/lib/util/service/log/package.go b/lib/util/service/log/package.go new file mode 100644 index 0000000..befa08f --- /dev/null +++ b/lib/util/service/log/package.go @@ -0,0 +1,34 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* +Package log - Some utilities for logging and stats aggregation/pushing. This package wraps third +party libraries in agnostic API calls so they can be swapped. +*/ +package log + +import "errors" + +// Errors used throughout the package. +var ( + ErrClientNil = errors.New("the client pointer was nil") +) diff --git a/lib/util/service/metrics/constructor.go b/lib/util/service/metrics/constructor.go new file mode 100644 index 0000000..9694267 --- /dev/null +++ b/lib/util/service/metrics/constructor.go @@ -0,0 +1,111 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +import ( + "bytes" + "errors" + "sort" + "strings" +) + +//-------------------------------------------------------------------------------------------------- + +// Errors for the metrics package. +var ( + ErrInvalidMetricOutputType = errors.New("invalid metrics output type") +) + +//-------------------------------------------------------------------------------------------------- + +// typeSpec - Constructor and a usage description for each metric output type. +type typeSpec struct { + constructor func(conf Config) (Type, error) + description string +} + +var constructors = map[string]typeSpec{} + +//-------------------------------------------------------------------------------------------------- + +// Config - The all encompassing configuration struct for all metric output types. +type Config struct { + Type string `json:"type" yaml:"type"` + HTTP HTTPConfig `json:"http_server" yaml:"http_server"` + Riemann RiemannConfig `json:"riemann" yaml:"riemann"` + Statsd StatsdConfig `json:"statsd" yaml:"statsd"` +} + +// NewConfig - Returns a configuration struct fully populated with default values. +func NewConfig() Config { + return Config{ + Type: "none", + HTTP: NewHTTPConfig(), + Riemann: NewRiemannConfig(), + Statsd: NewStatsdConfig(), + } +} + +//-------------------------------------------------------------------------------------------------- + +// Descriptions - Returns a formatted string of collated descriptions of each type. +func Descriptions() string { + // Order our input types alphabetically + names := []string{} + for name := range constructors { + names = append(names, name) + } + sort.Strings(names) + + buf := bytes.Buffer{} + buf.WriteString("METRIC TARGETS\n") + buf.WriteString(strings.Repeat("=", 80)) + buf.WriteString("\n\n") + + // Append each description + for i, name := range names { + buf.WriteString(name) + buf.WriteString("\n") + buf.WriteString(strings.Repeat("-", 80)) + buf.WriteString("\n") + buf.WriteString(constructors[name].description) + buf.WriteString("\n") + if i != (len(names) - 1) { + buf.WriteString("\n") + } + } + return buf.String() +} + +// New - Create a metric output type based on a configuration. +func New(conf Config) (Type, error) { + if conf.Type == "none" { + return DudType{}, nil + } + if c, ok := constructors[conf.Type]; ok { + return c.constructor(conf) + } + return nil, ErrInvalidMetricOutputType +} + +//-------------------------------------------------------------------------------------------------- diff --git a/lib/util/service/metrics/constructor_test.go b/lib/util/service/metrics/constructor_test.go new file mode 100644 index 0000000..1653286 --- /dev/null +++ b/lib/util/service/metrics/constructor_test.go @@ -0,0 +1,35 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +import "testing" + +func TestInterfaces(t *testing.T) { + foo, err := New(NewConfig()) + if err != nil { + t.Error(err) + } + bar := Type(foo) + foo.Incr("nope", 1) + bar.Incr("nope", 1) +} diff --git a/lib/util/service/metrics/dud_type.go b/lib/util/service/metrics/dud_type.go new file mode 100644 index 0000000..e3c40e0 --- /dev/null +++ b/lib/util/service/metrics/dud_type.go @@ -0,0 +1,45 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +//-------------------------------------------------------------------------------------------------- + +// DudType - Implements the Type interface but doesn't actual do anything. +type DudType struct{} + +// Incr - Does nothing. +func (d DudType) Incr(path string, count int64) error { return nil } + +// Decr - Does nothing. +func (d DudType) Decr(path string, count int64) error { return nil } + +// Timing - Does nothing. +func (d DudType) Timing(path string, delta int64) error { return nil } + +// Gauge - Does nothing. +func (d DudType) Gauge(path string, value int64) error { return nil } + +// Close - Does nothing. +func (d DudType) Close() error { return nil } + +//-------------------------------------------------------------------------------------------------- diff --git a/lib/util/service/metrics/dud_type_test.go b/lib/util/service/metrics/dud_type_test.go new file mode 100644 index 0000000..2453b6e --- /dev/null +++ b/lib/util/service/metrics/dud_type_test.go @@ -0,0 +1,32 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +import "testing" + +func TestDudInterface(t *testing.T) { + d := DudType{} + if Type(d) == nil { + t.Errorf("DudType does not satisfy Type interface.") + } +} diff --git a/lib/util/service/metrics/http.go b/lib/util/service/metrics/http.go new file mode 100644 index 0000000..93ef50c --- /dev/null +++ b/lib/util/service/metrics/http.go @@ -0,0 +1,187 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +import ( + "errors" + "fmt" + "net/http" + "runtime" + "sync" + "time" + + "github.com/Jeffail/gabs" +) + +//-------------------------------------------------------------------------------------------------- + +func init() { + constructors["http_server"] = typeSpec{ + constructor: NewHTTP, + description: ` +Benthos can host its own stats endpoint, where a GET request will receive a JSON +blob of all metrics tracked within Benthos.`, + } +} + +//-------------------------------------------------------------------------------------------------- + +// Errors for the HTTP type. +var ( + ErrTimedOut = errors.New("timed out") +) + +//-------------------------------------------------------------------------------------------------- + +// HTTPConfig - Config for the HTTP metrics type. +type HTTPConfig struct { + Prefix string `json:"stats_prefix" yaml:"stats_prefix"` + Address string `json:"address" yaml:"address"` + Path string `json:"path" yaml:"path"` +} + +// NewHTTPConfig - Creates an HTTPConfig struct with default values. +func NewHTTPConfig() HTTPConfig { + return HTTPConfig{ + Prefix: "service", + Address: "localhost:4040", + Path: "/stats", + } +} + +//-------------------------------------------------------------------------------------------------- + +// HTTP - A stats object with capability to hold internal stats as a JSON endpoint. +type HTTP struct { + config HTTPConfig + jsonRoot *gabs.Container + json *gabs.Container + flatMetrics map[string]int64 + pathPrefix string + timestamp time.Time + + sync.Mutex +} + +// NewHTTP - Create and return a new HTTP object. +func NewHTTP(config Config) (Type, error) { + var jsonRoot, json *gabs.Container + var pathPrefix string + + jsonRoot = gabs.New() + if len(config.HTTP.Prefix) > 0 { + pathPrefix = config.HTTP.Prefix + "." + json, _ = jsonRoot.ObjectP(config.HTTP.Prefix) + } else { + json = jsonRoot + } + + t := &HTTP{ + config: config.HTTP, + jsonRoot: jsonRoot, + json: json, + flatMetrics: map[string]int64{}, + pathPrefix: pathPrefix, + timestamp: time.Now(), + } + + go func() { + mux := http.NewServeMux() + mux.HandleFunc(config.HTTP.Path, t.JSONHandler()) + + http.ListenAndServe(config.HTTP.Address, mux) + }() + + return t, nil +} + +//-------------------------------------------------------------------------------------------------- + +// JSONHandler - Returns a handler for accessing metrics as a JSON blob. +func (h *HTTP) JSONHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + uptime := time.Since(h.timestamp).String() + goroutines := runtime.NumGoroutine() + + h.Lock() + + h.json.SetP(fmt.Sprintf("%v", uptime), "uptime") + h.json.SetP(goroutines, "goroutines") + blob := h.jsonRoot.Bytes() + + h.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.Write(blob) + } +} + +// Incr - Increment a stat by a value. +func (h *HTTP) Incr(stat string, value int64) error { + h.Lock() + total, _ := h.flatMetrics[stat] + total += value + + h.flatMetrics[stat] = total + h.json.SetP(total, stat) + h.Unlock() + return nil +} + +// Decr - Decrement a stat by a value. +func (h *HTTP) Decr(stat string, value int64) error { + h.Lock() + total, _ := h.flatMetrics[stat] + total -= value + + h.flatMetrics[stat] = total + h.json.SetP(total, stat) + h.Unlock() + return nil +} + +// Timing - Set a stat representing a duration. +func (h *HTTP) Timing(stat string, delta int64) error { + readable := time.Duration(delta).String() + + h.Lock() + h.json.SetP(delta, stat) + h.json.SetP(readable, stat+"_readable") + h.Unlock() + return nil +} + +// Gauge - Set a stat as a gauge value. +func (h *HTTP) Gauge(stat string, value int64) error { + h.Lock() + h.json.SetP(value, stat) + h.Unlock() + return nil +} + +// Close - Stops the HTTP object from aggregating metrics and cleans up resources. +func (h *HTTP) Close() error { + return nil +} + +//-------------------------------------------------------------------------------------------------- diff --git a/lib/util/service/metrics/http_test.go b/lib/util/service/metrics/http_test.go new file mode 100644 index 0000000..5e51dda --- /dev/null +++ b/lib/util/service/metrics/http_test.go @@ -0,0 +1,32 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +import "testing" + +func TestHTTPInterface(t *testing.T) { + o := &HTTP{} + if Type(o) == nil { + t.Errorf("Type does not satisfy Type interface.") + } +} diff --git a/lib/util/service/metrics/interface.go b/lib/util/service/metrics/interface.go new file mode 100644 index 0000000..8be32d4 --- /dev/null +++ b/lib/util/service/metrics/interface.go @@ -0,0 +1,41 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +// Type - An interface for metrics aggregation. +type Type interface { + // Incr - Increment a metric by an amount. + Incr(path string, count int64) error + + // Decr - Decrement a metric by an amount. + Decr(path string, count int64) error + + // Timing - Set a timing metric. + Timing(path string, delta int64) error + + // Gauge - Set a gauge metric. + Gauge(path string, value int64) error + + // Close - Stop aggregating stats and clean up resources. + Close() error +} diff --git a/lib/util/service/metrics/package.go b/lib/util/service/metrics/package.go new file mode 100644 index 0000000..c947a34 --- /dev/null +++ b/lib/util/service/metrics/package.go @@ -0,0 +1,39 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/* +Package metrics - Create a type for aggregating and propagating metrics to various services based on +configuration. Use it like this: + +``` go +conf := metrics.NewConfig() +conf.Type = "http_server" + +met, err := metrics.New(conf) +if err != nil { + panic(err) +} + +met.Incr("path.to.metric", 1) +``` +*/ +package metrics diff --git a/lib/util/service/metrics/riemann.go b/lib/util/service/metrics/riemann.go new file mode 100644 index 0000000..48a853f --- /dev/null +++ b/lib/util/service/metrics/riemann.go @@ -0,0 +1,225 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +import ( + "fmt" + "sync" + "time" + + "github.com/amir/raidman" +) + +//-------------------------------------------------------------------------------------------------- + +func init() { + constructors["riemann"] = typeSpec{ + constructor: NewRiemann, + description: ` +Benthos can send metrics to Riemann as events, you can set your own tags but it +is recommended that you ensure the 'meter' tag is there to ensure they are dealt +with correctly within Riemann.`, + } +} + +//-------------------------------------------------------------------------------------------------- + +// RiemannConfig - Configuration fields for a riemann service. +type RiemannConfig struct { + Server string `json:"server" yaml:"server"` + TTL float32 `json:"ttl" yaml:"ttl"` + Tags []string `json:"tags" yaml:"tags"` + FlushInterval string `json:"flush_interval" yaml:"flush_interval"` + Prefix string `json:"prefix" yaml:"prefix"` +} + +// NewRiemannConfig - Create a new riemann config with default values. +func NewRiemannConfig() RiemannConfig { + return RiemannConfig{ + Server: "", + TTL: 5, + Tags: []string{"service", "meter"}, + FlushInterval: "2s", + Prefix: "", + } +} + +//-------------------------------------------------------------------------------------------------- + +// Riemann - A Riemann client that supports the Type interface. +type Riemann struct { + sync.Mutex + + config RiemannConfig + + flatMetrics map[string]int64 + + Client *raidman.Client + eventsCache map[string]*raidman.Event + + flushInterval time.Duration + quit chan bool +} + +// NewRiemann - Create a new riemann client. +func NewRiemann(config Config) (Type, error) { + interval, err := time.ParseDuration(config.Riemann.FlushInterval) + if nil != err { + return nil, fmt.Errorf("failed to parse flush interval: %v", err) + } + + client, err := raidman.Dial("tcp", config.Riemann.Server) + if err != nil { + return nil, err + } + + r := &Riemann{ + config: config.Riemann, + Client: client, + flushInterval: interval, + eventsCache: make(map[string]*raidman.Event), + quit: make(chan bool), + } + + go r.loop() + + return r, nil +} + +//-------------------------------------------------------------------------------------------------- + +// Incr - Increment a stat by a value. +func (r *Riemann) Incr(stat string, value int64) error { + r.Lock() + defer r.Unlock() + + total, _ := r.flatMetrics[stat] + total += value + + r.flatMetrics[stat] = total + + service := r.config.Prefix + stat + r.eventsCache[service] = &raidman.Event{ + Ttl: r.config.TTL, + Tags: r.config.Tags, + Metric: total, + Service: service, + } + return nil +} + +// Decr - Decrement a stat by a value. +func (r *Riemann) Decr(stat string, value int64) error { + r.Lock() + defer r.Unlock() + + total, _ := r.flatMetrics[stat] + total -= value + + r.flatMetrics[stat] = total + + service := r.config.Prefix + stat + r.eventsCache[service] = &raidman.Event{ + Ttl: r.config.TTL, + Tags: r.config.Tags, + Metric: total, + Service: service, + } + return nil +} + +// Timing - Set a stat representing a duration. +func (r *Riemann) Timing(stat string, delta int64) error { + r.Lock() + defer r.Unlock() + + service := r.config.Prefix + stat + r.eventsCache[service] = &raidman.Event{ + Ttl: r.config.TTL, + Tags: r.config.Tags, + Metric: delta, + Service: service, + } + return nil +} + +// Gauge - Set a stat as a gauge value. +func (r *Riemann) Gauge(stat string, value int64) error { + r.Lock() + defer r.Unlock() + + service := r.config.Prefix + stat + r.eventsCache[service] = &raidman.Event{ + Ttl: r.config.TTL, + Tags: r.config.Tags, + Metric: value, + Service: service, + } + return nil +} + +// Close - Close the riemann client and stop batch uploading. +func (r *Riemann) Close() error { + close(r.quit) + return nil +} + +//-------------------------------------------------------------------------------------------------- + +func (r *Riemann) loop() { + ticker := time.NewTicker(r.flushInterval) + for { + select { + case <-ticker.C: + r.flushMetrics() + case <-r.quit: + r.Client.Close() + return + } + } +} + +func (r *Riemann) flushMetrics() { + r.Lock() + defer r.Unlock() + + events := make([]*raidman.Event, len(r.eventsCache)) + i := 0 + for _, event := range r.eventsCache { + events[i] = event + i++ + } + + if err := r.Client.SendMulti(events); err == nil { + r.eventsCache = make(map[string]*raidman.Event) + } else { + var newClient *raidman.Client + newClient, err = raidman.DialWithTimeout("tcp", r.config.Server, r.flushInterval) + if err == nil { + r.Client.Close() + r.Client = newClient + } + } +} + +//-------------------------------------------------------------------------------------------------- diff --git a/lib/util/service/metrics/statsd.go b/lib/util/service/metrics/statsd.go new file mode 100644 index 0000000..bc2e23c --- /dev/null +++ b/lib/util/service/metrics/statsd.go @@ -0,0 +1,125 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package metrics + +import ( + "fmt" + "time" + + "gopkg.in/alexcesaro/statsd.v2" +) + +//-------------------------------------------------------------------------------------------------- + +func init() { + constructors["statsd"] = typeSpec{ + constructor: NewStatsd, + description: `Use the statsd protocol.`, + } +} + +//-------------------------------------------------------------------------------------------------- + +// StatsdConfig - Config for the Statsd metrics type. +type StatsdConfig struct { + Address string `json:"address" yaml:"address"` + FlushPeriod string `json:"flush_period" yaml:"flush_period"` + MaxPacketSize int `json:"max_packet_size" yaml:"max_packet_size"` + Network string `json:"network" yaml:"network"` + Prefix string `json:"prefix" yaml:"prefix"` +} + +// NewStatsdConfig - Creates an StatsdConfig struct with default values. +func NewStatsdConfig() StatsdConfig { + return StatsdConfig{ + Address: "localhost:4040", + FlushPeriod: "100ms", + MaxPacketSize: 1440, + Network: "udp", + Prefix: "", + } +} + +//-------------------------------------------------------------------------------------------------- + +// Statsd - A stats object with capability to hold internal stats as a JSON endpoint. +type Statsd struct { + config Config + s *statsd.Client +} + +// NewStatsd - Create and return a new Statsd object. +func NewStatsd(config Config) (Type, error) { + flushPeriod, err := time.ParseDuration(config.Statsd.FlushPeriod) + if err != nil { + return nil, fmt.Errorf("Failed to parse flush period: %s", err) + } + c, err := statsd.New( + statsd.Address(config.Statsd.Address), + statsd.FlushPeriod(flushPeriod), + statsd.MaxPacketSize(config.Statsd.MaxPacketSize), + statsd.Network(config.Statsd.Network), + statsd.Prefix(config.Statsd.Prefix), + ) + if err != nil { + return nil, err + } + return &Statsd{ + config: config, + s: c, + }, nil +} + +//-------------------------------------------------------------------------------------------------- + +// Incr - Increment a stat by a value. +func (h *Statsd) Incr(stat string, value int64) error { + h.s.Count(stat, value) + return nil +} + +// Decr - Decrement a stat by a value. +func (h *Statsd) Decr(stat string, value int64) error { + h.s.Count(stat, -value) + return nil +} + +// Timing - Set a stat representing a duration. +func (h *Statsd) Timing(stat string, delta int64) error { + h.s.Timing(stat, delta) + return nil +} + +// Gauge - Set a stat as a gauge value. +func (h *Statsd) Gauge(stat string, value int64) error { + h.s.Gauge(stat, value) + return nil +} + +// Close - Stops the Statsd object from aggregating metrics and cleans up resources. +func (h *Statsd) Close() error { + h.s.Close() + return nil +} + +//-------------------------------------------------------------------------------------------------- diff --git a/lib/util/service/path/fromBinary.go b/lib/util/service/path/fromBinary.go new file mode 100644 index 0000000..de5eed2 --- /dev/null +++ b/lib/util/service/path/fromBinary.go @@ -0,0 +1,61 @@ +/* +Copyright (c) 2014 Ashley Jeffs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, sub to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package path + +import ( + "path/filepath" + + "github.com/kardianos/osext" +) + +var ( + executablePath string + resolveError error +) + +func init() { + // Get the location of the executing binary + executablePath, resolveError = osext.ExecutableFolder() +} + +/* +FromBinaryIfRelative - Takes a path and, if the path is relative, resolves the path from the +location of the binary file rather than the current working directory. Returns an error when the +path is relative and cannot be resolved. +*/ +func FromBinaryIfRelative(path *string) error { + if !filepath.IsAbs(*path) { + if resolveError != nil { + return resolveError + } + *path = filepath.Join(executablePath, *path) + } + return nil +} + +/* +BinaryPath - Returns the path of the executing binary, or an error if it couldn't be resolved. +*/ +func BinaryPath() (string, error) { + return executablePath, resolveError +} diff --git a/vendor/github.com/amir/raidman b/vendor/github.com/amir/raidman deleted file mode 160000 index 91c20f3..0000000 --- a/vendor/github.com/amir/raidman +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 91c20f3f475cab75bb40ad7951d9bbdde357ade7 diff --git a/vendor/github.com/codemirror/codemirror b/vendor/github.com/codemirror/codemirror deleted file mode 160000 index a66bcf5..0000000 --- a/vendor/github.com/codemirror/codemirror +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a66bcf59d959b2ef724f80080f8d128d9dcf318e diff --git a/vendor/github.com/elazarl/go-bindata-assetfs b/vendor/github.com/elazarl/go-bindata-assetfs deleted file mode 160000 index 57eb5e1..0000000 --- a/vendor/github.com/elazarl/go-bindata-assetfs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 57eb5e1fc594ad4b0b1dbea7b286d299e0cb43c2 diff --git a/vendor/github.com/garyburd/redigo b/vendor/github.com/garyburd/redigo deleted file mode 160000 index 8873b2f..0000000 --- a/vendor/github.com/garyburd/redigo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8873b2f1995f59d4bcdd2b0dc9858e2cb9bf0c13 diff --git a/vendor/github.com/go-sql-driver/mysql b/vendor/github.com/go-sql-driver/mysql deleted file mode 160000 index 7ebe0a5..0000000 --- a/vendor/github.com/go-sql-driver/mysql +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ebe0a500653eeb1859664bed5e48dec1e164e73 diff --git a/vendor/github.com/golang/protobuf b/vendor/github.com/golang/protobuf deleted file mode 160000 index bf531ff..0000000 --- a/vendor/github.com/golang/protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bf531ff1a004f24ee53329dfd5ce0b41bfdc17df diff --git a/vendor/github.com/gorilla/websocket b/vendor/github.com/gorilla/websocket deleted file mode 160000 index a91eba7..0000000 --- a/vendor/github.com/gorilla/websocket +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a91eba7f97777409bc2c443f5534d41dd20c5720 diff --git a/vendor/github.com/jeffail/gabs b/vendor/github.com/jeffail/gabs deleted file mode 160000 index ee1575a..0000000 --- a/vendor/github.com/jeffail/gabs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ee1575a53249b51d636e62464ca43a13030afdb5 diff --git a/vendor/github.com/jeffail/util b/vendor/github.com/jeffail/util deleted file mode 160000 index 36e8827..0000000 --- a/vendor/github.com/jeffail/util +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 36e882756e538c55352881516b3796ed6399f4cb diff --git a/vendor/github.com/jteeuwen/go-bindata b/vendor/github.com/jteeuwen/go-bindata deleted file mode 160000 index a0ff256..0000000 --- a/vendor/github.com/jteeuwen/go-bindata +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a0ff2567cfb70903282db057e799fd826784d41d diff --git a/vendor/github.com/lib/pq b/vendor/github.com/lib/pq deleted file mode 160000 index 3cd0097..0000000 --- a/vendor/github.com/lib/pq +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3cd0097429be7d611bb644ef85b42bfb102ceea4 diff --git a/vendor/github.com/satori/go.uuid b/vendor/github.com/satori/go.uuid deleted file mode 160000 index f9ab0dc..0000000 --- a/vendor/github.com/satori/go.uuid +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f9ab0dce87d815821e221626b772e3475a0d2749 diff --git a/vendor/github.com/vuejs/vue b/vendor/github.com/vuejs/vue deleted file mode 160000 index 8d56a49..0000000 --- a/vendor/github.com/vuejs/vue +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8d56a498f3d95660e2cafff4ed3c1e1da23494d1 diff --git a/vendor/golang.org/x/net b/vendor/golang.org/x/net deleted file mode 160000 index 3405706..0000000 --- a/vendor/golang.org/x/net +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 34057069f4ab13dc4433c68d368737ebeafcccdc