From 95ddacb9f308c5e4f598b081c0e6b93c91511c92 Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Tue, 24 Dec 2024 00:15:10 -0300 Subject: [PATCH] sdk: track query attempts automatically and other small tweaks to replaceable fetching and stuff. --- pool.go | 42 ++++++++++++++++++++++++++++++++++----- sdk/replaceable_loader.go | 12 +++++++---- sdk/system.go | 8 +++++--- sdk/tracker.go | 27 ++++++++++++++++++++++--- 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/pool.go b/pool.go index 65e70c8..743f3bd 100644 --- a/pool.go +++ b/pool.go @@ -25,7 +25,8 @@ type SimplePool struct { authHandler func(context.Context, RelayEvent) error cancel context.CancelFunc - eventMiddleware []func(RelayEvent) + eventMiddleware func(RelayEvent) + queryMiddleware func(relay string, pubkey string, kind int) // custom things not often used penaltyBoxMu sync.Mutex @@ -114,11 +115,18 @@ func (h withPenaltyBoxOpt) ApplyPoolOption(pool *SimplePool) { } // WithEventMiddleware is a function that will be called with all events received. -// more than one can be passed at a time. type WithEventMiddleware func(RelayEvent) func (h WithEventMiddleware) ApplyPoolOption(pool *SimplePool) { - pool.eventMiddleware = append(pool.eventMiddleware, h) + pool.eventMiddleware = h +} + +// WithQueryMiddleware is a function that will be called with every combination of relay+pubkey+kind queried +// in a .SubMany*() call -- when applicable (i.e. when the query contains a pubkey and a kind). +type WithQueryMiddleware func(relay string, pubkey string, kind int) + +func (h WithQueryMiddleware) ApplyPoolOption(pool *SimplePool) { + pool.queryMiddleware = h } // WithUserAgent sets the user-agent header for all relay connections in the pool. @@ -271,6 +279,18 @@ func (pool *SimplePool) subMany( var sub *Subscription + if mh := pool.queryMiddleware; mh != nil { + for _, filter := range filters { + if filter.Kinds != nil && filter.Authors != nil { + for _, kind := range filter.Kinds { + for _, author := range filter.Authors { + mh(nm, author, kind) + } + } + } + } + } + relay, err := pool.EnsureRelay(nm) if err != nil { goto reconnect @@ -306,7 +326,7 @@ func (pool *SimplePool) subMany( } ie := RelayEvent{Event: evt, Relay: relay} - for _, mh := range pool.eventMiddleware { + if mh := pool.eventMiddleware; mh != nil { mh(ie) } @@ -407,6 +427,18 @@ func (pool *SimplePool) subManyEose( go func(nm string) { defer wg.Done() + if mh := pool.queryMiddleware; mh != nil { + for _, filter := range filters { + if filter.Kinds != nil && filter.Authors != nil { + for _, kind := range filter.Kinds { + for _, author := range filter.Authors { + mh(nm, author, kind) + } + } + } + } + } + relay, err := pool.EnsureRelay(nm) if err != nil { return @@ -446,7 +478,7 @@ func (pool *SimplePool) subManyEose( } ie := RelayEvent{Event: evt, Relay: relay} - for _, mh := range pool.eventMiddleware { + if mh := pool.eventMiddleware; mh != nil { mh(ie) } diff --git a/sdk/replaceable_loader.go b/sdk/replaceable_loader.go index 334dacc..91fc962 100644 --- a/sdk/replaceable_loader.go +++ b/sdk/replaceable_loader.go @@ -124,15 +124,19 @@ func (sys *System) determineRelaysToQuery(ctx context.Context, pubkey string, ki if kind == 10002 { // prevent infinite loops by jumping directly to this relays = sys.Hints.TopN(pubkey, 3) - } else if kind == 0 { - // leave room for one hardcoded relay because people are stupid - relays = sys.FetchOutboxRelays(ctx, pubkey, 2) + if len(relays) == 0 { + relays = []string{"wss://relay.damus.io", "wss://nos.lol"} + } + } else if kind == 0 || kind == 3 { + // leave room for two hardcoded relays because people are stupid + relays = sys.FetchOutboxRelays(ctx, pubkey, 1) } else { relays = sys.FetchOutboxRelays(ctx, pubkey, 3) } // use a different set of extra relays depending on the kind - for i := 0; i < 3-len(relays); i++ { + needed := 3 - len(relays) + for range needed { var next string switch kind { case 0: diff --git a/sdk/system.go b/sdk/system.go index b3b084f..52e964a 100644 --- a/sdk/system.go +++ b/sdk/system.go @@ -2,6 +2,7 @@ package sdk import ( "context" + "math/rand/v2" "github.com/fiatjaf/eventstore" "github.com/fiatjaf/eventstore/nullstore" @@ -10,7 +11,7 @@ import ( "github.com/nbd-wtf/go-nostr/sdk/cache" cache_memory "github.com/nbd-wtf/go-nostr/sdk/cache/memory" "github.com/nbd-wtf/go-nostr/sdk/hints" - memory_hints "github.com/nbd-wtf/go-nostr/sdk/hints/memory" + "github.com/nbd-wtf/go-nostr/sdk/hints/memoryh" ) type System struct { @@ -43,7 +44,7 @@ type RelayStream struct { } func NewRelayStream(urls ...string) *RelayStream { - return &RelayStream{URLs: urls, serial: -1} + return &RelayStream{URLs: urls, serial: rand.Int()} } func (rs *RelayStream) Next() string { @@ -82,12 +83,13 @@ func NewSystem(mods ...SystemModifier) *System { "wss://relay.nostr.band", "wss://search.nos.today", ), - Hints: memory_hints.NewHintDB(), + Hints: memoryh.NewHintDB(), outboxShortTermCache: cache_memory.New32[[]string](1000), } sys.Pool = nostr.NewSimplePool(context.Background(), + nostr.WithQueryMiddleware(sys.TrackQueryAttempts), nostr.WithEventMiddleware(sys.TrackEventHints), nostr.WithPenaltyBox(), ) diff --git a/sdk/tracker.go b/sdk/tracker.go index 90cf6a9..74827df 100644 --- a/sdk/tracker.go +++ b/sdk/tracker.go @@ -7,6 +7,19 @@ import ( "github.com/nbd-wtf/go-nostr/sdk/hints" ) +func (sys *System) TrackQueryAttempts(relay string, author string, kind int) { + if IsVirtualRelay(relay) { + return + } + if kind < 30000 && kind >= 20000 { + return + } + if kind == 0 || kind == 10002 || kind == 3 { + return + } + sys.Hints.Save(author, relay, hints.LastFetchAttempt, nostr.Now()) +} + func (sys *System) TrackEventHints(ie nostr.RelayEvent) { if IsVirtualRelay(ie.Relay.URL) { return @@ -16,7 +29,15 @@ func (sys *System) TrackEventHints(ie nostr.RelayEvent) { } switch ie.Kind { + case nostr.KindProfileMetadata: + // this could be anywhere so it doesn't count + return case nostr.KindRelayListMetadata: + // this is special, we only use it to track relay-list hints + if len(ie.Tags) > 12 { + // too many relays in the list means this person is not using this correctly so we better ignore them + return + } for _, tag := range ie.Tags { if len(tag) < 2 || tag[0] != "r" { continue @@ -26,8 +47,7 @@ func (sys *System) TrackEventHints(ie nostr.RelayEvent) { } } case nostr.KindFollowList: - sys.Hints.Save(ie.PubKey, ie.Relay.URL, hints.MostRecentEventFetched, ie.CreatedAt) - + // this is special, we only use it to check if there are hints for the contacts for _, tag := range ie.Tags { if len(tag) < 3 { continue @@ -42,7 +62,8 @@ func (sys *System) TrackEventHints(ie nostr.RelayEvent) { sys.Hints.Save(tag[1], tag[2], hints.LastInTag, ie.CreatedAt) } } - case nostr.KindTextNote: + default: + // everything else may have hints sys.Hints.Save(ie.PubKey, ie.Relay.URL, hints.MostRecentEventFetched, ie.CreatedAt) for _, tag := range ie.Tags {