From c397f96969ba62ec245ace37c84bcddc8d37a0aa Mon Sep 17 00:00:00 2001 From: igolaizola <11333576+igolaizola@users.noreply.github.com> Date: Thu, 6 Jun 2024 23:55:14 +0200 Subject: [PATCH] Add support for refreshing Discord CDN URLs This change adds support for refreshing Discord CDN URLs in a file. It introduces a new command `refresh` in the `bulkai` CLI tool. The `refresh` command allows users to update the CDN URLs of attachments in a file by making requests to the Discord API. The `refresh` command accepts various flags such as `proxy`, `input`, `output`, `wait`, and `debug` to customize the refresh process. Additionally, it uses a session configuration file to provide user agent, JA3 fingerprint, language, authentication token, super properties, locale, and cookie information. This feature aims to improve the reliability of image URLs by refreshing them periodically. It provides an alternative to using the Midjourney CDN for those who still prefer using Discord CDN URLs. --- cmd/bulkai/main.go | 45 +++++++++ pkg/cmd/refresh/refresh.go | 195 +++++++++++++++++++++++++++++++++++++ pkg/discord/discord.go | 17 +++- 3 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 pkg/cmd/refresh/refresh.go diff --git a/cmd/bulkai/main.go b/cmd/bulkai/main.go index 934dbd0..580ad5b 100644 --- a/cmd/bulkai/main.go +++ b/cmd/bulkai/main.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/igolaizola/bulkai" + "github.com/igolaizola/bulkai/pkg/cmd/refresh" "github.com/igolaizola/bulkai/pkg/session" "github.com/peterbourgon/ff/v3" "github.com/peterbourgon/ff/v3/ffcli" @@ -46,6 +47,7 @@ func newCommand() *ffcli.Command { Subcommands: []*ffcli.Command{ newGenerateCommand(), newCreateSessionCommand(), + newRefreshCommand(), newVersionCommand(), }, } @@ -116,6 +118,49 @@ func newGenerateCommand() *ffcli.Command { } } +func newRefreshCommand() *ffcli.Command { + fs := flag.NewFlagSet("refresh", flag.ExitOnError) + _ = fs.String("config", "", "config file (optional)") + + cfg := &refresh.Config{} + + fs.StringVar(&cfg.Proxy, "proxy", "", "proxy address (optional)") + fs.StringVar(&cfg.Input, "input", "input", "input file") + fs.StringVar(&cfg.Output, "output", "output", "output file") + fs.DurationVar(&cfg.Wait, "wait", 0, "wait time between requests (optional)") + fs.BoolVar(&cfg.Debug, "debug", false, "debug mode") + + // Session + fs.StringVar(&cfg.SessionFile, "session", "session.yaml", "session config file (optional)") + + fsSession := flag.NewFlagSet("", flag.ExitOnError) + for _, fs := range []*flag.FlagSet{fs, fsSession} { + fs.StringVar(&cfg.Session.UserAgent, "user-agent", "", "user agent") + fs.StringVar(&cfg.Session.JA3, "ja3", "", "ja3 fingerprint") + fs.StringVar(&cfg.Session.Language, "language", "", "language") + fs.StringVar(&cfg.Session.Token, "token", "", "authentication token") + fs.StringVar(&cfg.Session.SuperProperties, "super-properties", "", "super properties") + fs.StringVar(&cfg.Session.Locale, "locale", "", "locale") + fs.StringVar(&cfg.Session.Cookie, "cookie", "", "cookie") + } + + return &ffcli.Command{ + Name: "refresh", + ShortUsage: "bulkai refresh [flags] ", + Options: []ff.Option{ + ff.WithConfigFileFlag("config"), + ff.WithConfigFileParser(ffyaml.Parser), + ff.WithEnvVarPrefix("BULKAI"), + }, + ShortHelp: "refresh discord CDN URLs in a file", + FlagSet: fs, + Exec: func(ctx context.Context, args []string) error { + loadSession(fsSession, cfg.SessionFile) + return refresh.Run(ctx, cfg) + }, + } +} + func newCreateSessionCommand() *ffcli.Command { fs := flag.NewFlagSet("create-session", flag.ExitOnError) _ = fs.String("config", "", "config file (optional)") diff --git a/pkg/cmd/refresh/refresh.go b/pkg/cmd/refresh/refresh.go new file mode 100644 index 0000000..f59d108 --- /dev/null +++ b/pkg/cmd/refresh/refresh.go @@ -0,0 +1,195 @@ +package refresh + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/igolaizola/bulkai/pkg/discord" + "github.com/igolaizola/bulkai/pkg/http" + "gopkg.in/yaml.v2" +) + +type Config struct { + Debug bool `yaml:"debug"` + Proxy string `yaml:"proxy"` + Wait time.Duration `yaml:"wait"` + Input string `yaml:"input"` + Output string `yaml:"output"` + SessionFile string `yaml:"session"` + Session Session `yaml:"-"` +} + +type Session struct { + JA3 string `yaml:"ja3"` + UserAgent string `yaml:"user-agent"` + Language string `yaml:"language"` + Token string `yaml:"token"` + SuperProperties string `yaml:"super-properties"` + Locale string `yaml:"locale"` + Cookie string `yaml:"cookie"` +} + +func Run(ctx context.Context, cfg *Config) error { + if cfg.Session.Token == "" { + return errors.New("missing token") + } + if cfg.Session.JA3 == "" { + return errors.New("missing ja3") + } + if cfg.Session.UserAgent == "" { + return errors.New("missing user agent") + } + if cfg.Session.Cookie == "" { + return errors.New("missing cookie") + } + if cfg.Session.Language == "" { + return errors.New("missing language") + } + if cfg.Input == "" { + return errors.New("missing input file") + } + if cfg.Output == "" { + return errors.New("missing output file") + } + + // Load input file + b, err := os.ReadFile(cfg.Input) + if err != nil { + return err + } + + // Create output directory + if err := os.MkdirAll(filepath.Dir(cfg.Output), 0755); err != nil { + return err + } + + // Find all CDN URLs + urls := cdnReg.FindAllString(string(b), -1) + if len(urls) == 0 { + log.Println("no URLs found") + return nil + } + + // Create a unique list of URLs + var unique []string + lookup := map[string]struct{}{} + for _, url := range urls { + if _, ok := lookup[url]; !ok { + lookup[url] = struct{}{} + unique = append(unique, url) + } + } + + // Create http client + httpClient, err := http.NewClient(cfg.Session.JA3, cfg.Session.UserAgent, cfg.Session.Language, cfg.Proxy) + if err != nil { + return fmt.Errorf("couldn't create http client: %w", err) + } + + // Set proxy + if cfg.Proxy != "" { + p := strings.TrimPrefix(cfg.Proxy, "http://") + p = strings.TrimPrefix(p, "https://") + os.Setenv("HTTPS_PROXY", p) + os.Setenv("HTTP_PROXY", p) + } + + if err := http.SetCookies(httpClient, "https://discord.com", cfg.Session.Cookie); err != nil { + return fmt.Errorf("couldn't set cookies: %w", err) + } + defer func() { + cookie, err := http.GetCookies(httpClient, "https://discord.com") + if err != nil { + log.Printf("couldn't get cookies: %v\n", err) + } + cfg.Session.Cookie = strings.ReplaceAll(cookie, "\n", "") + // TODO: save session to common method + data, err := yaml.Marshal(cfg.Session) + if err != nil { + log.Println(fmt.Errorf("couldn't marshal session: %w", err)) + } + if err := os.WriteFile(cfg.SessionFile, data, 0644); err != nil { + log.Println(fmt.Errorf("couldn't write session: %w", err)) + } + }() + + // Create discord client + client, err := discord.New(ctx, &discord.Config{ + Token: cfg.Session.Token, + SuperProperties: cfg.Session.SuperProperties, + Locale: cfg.Session.Locale, + UserAgent: cfg.Session.UserAgent, + HTTPClient: httpClient, + Debug: cfg.Debug, + }) + if err != nil { + return fmt.Errorf("couldn't create discord client: %w", err) + } + + // Start discord client + if err := client.Start(ctx); err != nil { + return fmt.Errorf("couldn't start discord client: %w", err) + } + + // Refresh URLs + wait := 10 * time.Millisecond + for _, cdnURL := range unique { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(wait): + } + wait = cfg.Wait + + // Refresh URL + u := "attachments/refresh-urls" + shortURL := strings.Split(cdnURL, "?")[0] + req := &refreshURLsRequest{ + AttachmentURLs: []string{shortURL}, + } + resp, err := client.Do(ctx, "POST", u, req) + if err != nil { + return fmt.Errorf("couldn't refresh URL (%s): %w", cdnURL, err) + } + var refreshResp refreshURLsResponse + if err := json.Unmarshal(resp, &refreshResp); err != nil { + return fmt.Errorf("couldn't unmarshal response %s: %w", string(resp), err) + } + if len(refreshResp.RefreshedURLs) == 0 { + return fmt.Errorf("no refreshed URLs found (%s)", cdnURL) + } + refURL := refreshResp.RefreshedURLs[0] + if refURL.Refreshed == "" { + return fmt.Errorf("no refreshed URL found (%s)", cdnURL) + } + // Replace URL in input file + b = []byte(strings.ReplaceAll(string(b), cdnURL, refURL.Refreshed)) + } + + // Save output file + if err := os.WriteFile(cfg.Output, b, 0644); err != nil { + return err + } + return nil +} + +var cdnReg = regexp.MustCompile(`https:\/\/cdn\.discordapp\.com\/attachments\/[^"\s,]+`) + +type refreshURLsRequest struct { + AttachmentURLs []string `json:"attachment_urls"` +} + +type refreshURLsResponse struct { + RefreshedURLs []struct { + Original string `json:"original"` + Refreshed string `json:"refreshed"` + } `json:"refreshed_urls"` +} diff --git a/pkg/discord/discord.go b/pkg/discord/discord.go index 1e5e9bd..4f4a9aa 100644 --- a/pkg/discord/discord.go +++ b/pkg/discord/discord.go @@ -468,7 +468,21 @@ func (c *Client) addHeaders(req *http.Request) { "x-discord-locale": {c.locale}, "x-super-properties": {c.superProperties.raw}, } - + case "/api/v9/attachments/refresh-urls": + req.Header = http.Header{ + "accept": {"*/*"}, + "accept-encoding": {"gzip, deflate, br"}, + "authorization": {c.token}, + "content-type": {"application/json"}, + "origin": {"https://discord.com"}, + "referer": {referer}, + "sec-fetch-dest": {"empty"}, + "sec-fetch-mode": {"cors"}, + "sec-fetch-site": {"same-origin"}, + "x-debug-options": {"bugReporterEnabled"}, + "x-discord-locale": {c.locale}, + "x-super-properties": {c.superProperties.raw}, + } default: req.Header = http.Header{ "accept": {"*/*"}, @@ -489,6 +503,7 @@ func (c *Client) addHeaders(req *http.Request) { "accept", "accept-encoding", "accept-language", + "content-type", "cookie", "origin", "referer",