feat: add api endpoint to get total count of torrents based on query …

…keyword

The `/api/v0.1/torrentstotal` endpoint is used to get the total count of queried torrents.

When querying torrents through the `/api/v0.1/torrents` endpoint, it only returns the data for the current page without specifying how many total records there are.

The web interface, pagination is implemented by clicking "Load More". With this new endpoint, the total number of matching records can be fetched, making it easier to implement pagination in the web interface.

This will enable an implementation for pagination in the web query interface, no longer relying on 'id' for page navigation.

persistence/bitmagnet.go

@@ -125,6 +125,10 @@ func (b *bitmagnet) GetNumberOfTorrents() (uint, error) {
 	return 0, nil
 }
 
+func (b *bitmagnet) GetNumberOfQueryTorrents(query string, epoch int64) (uint64, error) {
+	return 0, nil
+}
+
 func (b *bitmagnet) QueryTorrents(query string, epoch int64, orderBy OrderingCriteria, ascending bool, limit uint64, lastOrderedValue *float64, lastID *uint64) ([]TorrentMetadata, error) {
 	return nil, errors.New("query not supported")
 }

persistence/interface.go

@@ -17,6 +17,8 @@ type Database interface {
 	// GetNumberOfTorrents returns the number of torrents saved in the database. Might be an
 	// approximation.
 	GetNumberOfTorrents() (uint, error)
+	// GetNumberOfQueryTorrents returns the total number of data records in a fuzzy query.
+	GetNumberOfQueryTorrents(query string, epoch int64) (uint64, error)
 	// QueryTorrents returns @pageSize amount of torrents,
 	// * that are discovered before @discoveredOnBefore
 	// * that match the @query if it's not empty, else all torrents

persistence/postgres.go

@@ -167,6 +167,38 @@ func (db *postgresDatabase) GetNumberOfTorrents() (uint, error) {
 	}
 }
 
+func (db *postgresDatabase) GetNumberOfQueryTorrents(query string, epoch int64) (uint64, error) {
+
+	var querySkeleton = `SELECT COUNT(*)
+		FROM torrents
+		WHERE
+    		name ILIKE CONCAT('%',$1::text,'%') AND
+			discovered_on <= $2;
+	`
+
+	rows, err := db.conn.Query(querySkeleton, query, epoch)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+
+	if !rows.Next() {
+		return 0, errors.New("no rows returned from `SELECT COUNT(*) FROM torrents WHERE name ILIKE CONCAT('%%',$1::text,'%%') AND discovered_on <= $2;`")
+	}
+
+	var n *int64
+	if err = rows.Scan(&n); err != nil {
+		return 0, err
+	}
+
+	// If the database is empty (i.e. 0 entries in 'torrents') then the query will return nil.
+	if n == nil || *n < 0 {
+		return 0, nil
+	} else {
+		return uint64(*n), nil
+	}
+}
+
 func (db *postgresDatabase) QueryTorrents(
 	query string,
 	epoch int64,

persistence/postgres_test.go

@@ -155,6 +155,79 @@ func TestPostgresDatabase_GetNumberOfTorrents(t *testing.T) {
 	}
 }
 
+func TestPostgresDatabase_GetNumberOfQueryTorrents(t *testing.T) {
+	t.Parallel()
+
+	conn, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("An error '%s' was not expected when opening a stub database connection", err)
+	}
+	defer conn.Close()
+
+	pgDb := &postgresDatabase{conn: conn}
+
+	query := "test-query"
+	epoch := int64(1609459200) // 2021-01-01 00:00:00 UTC
+
+	rows := sqlmock.NewRows([]string{"count"}).AddRow(int64(10))
+	mock.ExpectQuery(`SELECT COUNT\(\*\) FROM torrents WHERE name ILIKE CONCAT\('%',\$1::text,'%'\) AND discovered_on <= \$2;`).
+		WithArgs(query, epoch).
+		WillReturnRows(rows)
+
+	result, err := pgDb.GetNumberOfQueryTorrents(query, epoch)
+
+	if err != nil {
+		t.Errorf("Expected no error, but got %v", err)
+	}
+
+	if result != uint64(10) {
+		t.Errorf("Expected result to be 10, but got %d", result)
+	}
+
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Errorf("There were unmet expectations: %s", err)
+	}
+
+	rows = sqlmock.NewRows([]string{"count"})
+	mock.ExpectQuery(`SELECT COUNT\(\*\) FROM torrents WHERE name ILIKE CONCAT\('%',\$1::text,'%'\) AND discovered_on <= \$2;`).
+		WithArgs(query, epoch).
+		WillReturnRows(rows)
+
+	result, err = pgDb.GetNumberOfQueryTorrents(query, epoch)
+
+	if err == nil {
+		t.Error("Expected an error, but got none")
+	}
+
+	if result != uint64(0) {
+		t.Errorf("Expected result to be 0, but got %d", result)
+	}
+
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Errorf("There were unmet expectations: %s", err)
+	}
+
+	rows = sqlmock.NewRows([]string{"count"}).AddRow(nil)
+	mock.ExpectQuery(`SELECT COUNT\(\*\) FROM torrents WHERE name ILIKE CONCAT\('%',\$1::text,'%'\) AND discovered_on <= \$2;`).
+		WithArgs(query, epoch).
+		WillReturnRows(rows)
+
+	result, err = pgDb.GetNumberOfQueryTorrents(query, epoch)
+
+	if err != nil {
+		t.Errorf("Expected no error, but got %v", err)
+	}
+
+	if result != uint64(0) {
+		t.Errorf("Expected result to be 0, but got %d", result)
+	}
+
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Errorf("There were unmet expectations: %s", err)
+	}
+
+}
+
 func TestPostgresDatabase_Close(t *testing.T) {
 	t.Parallel()
 

persistence/rabbitmq.go

@@ -136,6 +136,10 @@ func (r *rabbitMQ) GetNumberOfTorrents() (uint, error) {
 	return 0, nil
 }
 
+func (r *rabbitMQ) GetNumberOfQueryTorrents(query string, epoch int64) (uint64, error) {
+	return 0, nil
+}
+
 func (r *rabbitMQ) QueryTorrents(query string, epoch int64, orderBy OrderingCriteria, ascending bool, limit uint64, lastOrderedValue *float64, lastID *uint64) ([]TorrentMetadata, error) {
 	return nil, errors.New("query not supported")
 }

persistence/sqlite3.go

@@ -228,6 +228,36 @@ func (db *sqlite3Database) GetNumberOfTorrents() (uint, error) {
 	}
 }
 
+func (db *sqlite3Database) GetNumberOfQueryTorrents(query string, epoch int64) (uint64, error) {
+	var querySkeleton = `SELECT COUNT(*)
+	FROM torrents
+	WHERE
+    	LOWER(name) LIKE '%' || LOWER($1) || '%' AND
+    	discovered_on <= $2;
+	`
+	rows, err := db.conn.Query(querySkeleton, query, epoch)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+
+	if !rows.Next() {
+		return 0, fmt.Errorf("no rows returned from `SELECT COUNT(*) FROM torrents WHERE LOWER(name) LIKE '%%' || LOWER($1) || '%%' AND discovered_on <= $2;`")
+	}
+
+	var n *uint
+	if err = rows.Scan(&n); err != nil {
+		return 0, err
+	}
+
+	// If the database is empty (i.e. 0 entries in 'torrents') then the query will return nil.
+	if n == nil {
+		return 0, nil
+	} else {
+		return uint64(*n), nil
+	}
+}
+
 func (db *sqlite3Database) QueryTorrents(
 	query string,
 	epoch int64,

persistence/sqlite3_test.go

@@ -73,6 +73,69 @@ func Test_sqlite3Database_GetNumberOfTorrents(t *testing.T) {
 	}
 }
 
+func TestSqlite3Database_GetNumberOfQueryTorrents(t *testing.T) {
+	t.Parallel()
+	db := newDb(t)
+
+	// The database is empty, so the number of torrents for any query should be 0.
+	tests := []struct {
+		name    string
+		query   string
+		epoch   int64
+		want    uint64
+		wantErr bool
+	}{
+		{
+			name:    "Test Empty Query",
+			query:   "",
+			epoch:   0,
+			want:    0,
+			wantErr: false,
+		},
+		{
+			name:    "Test Simple Query",
+			query:   "test",
+			epoch:   0,
+			want:    0,
+			wantErr: false,
+		},
+		{
+			name:    "Test Query with Special Characters",
+			query:   "test!@#$%^&*()",
+			epoch:   0,
+			want:    0,
+			wantErr: false,
+		},
+		{
+			name:    "Test Query with Future Epoch",
+			query:   "test",
+			epoch:   32503680000, // January 1, 3000
+			want:    0,
+			wantErr: false,
+		},
+		{
+			name:    "Test Query with Past Epoch",
+			query:   "test",
+			epoch:   1000000000, // September 9, 2001
+			want:    0,
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := db.GetNumberOfQueryTorrents(tt.query, tt.epoch)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("sqlite3Database.GetNumberOfQueryTorrents() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("sqlite3Database.GetNumberOfQueryTorrents() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func Test_sqlite3Database_AddNewTorrent(t *testing.T) {
 	t.Parallel()
 	db := newDb(t)

persistence/zeromq.go

@@ -90,6 +90,10 @@ func (instance *zeromq) GetNumberOfTorrents() (uint, error) {
 	return 0, nil
 }
 
+func (instance *zeromq) GetNumberOfQueryTorrents(query string, epoch int64) (uint64, error) {
+	return 0, nil
+}
+
 func (instance *zeromq) QueryTorrents(
 	query string,
 	epoch int64,

persistence/zeromq_mock.go

@@ -34,6 +34,10 @@ func (instance *zeromq) GetNumberOfTorrents() (uint, error) {
 	return 0, nil
 }
 
+func (instance *zeromq) GetNumberOfQueryTorrents(query string, epoch int64) (uint64, error) {
+	return 0, nil
+}
+
 func (instance *zeromq) QueryTorrents(
 	query string,
 	epoch int64,

web/router.go

@@ -57,6 +57,7 @@ func makeRouter() *http.ServeMux {
 
 	router.HandleFunc("/api/v0.1/statistics", BasicAuth(apiStatistics))
 	router.HandleFunc("/api/v0.1/torrents", BasicAuth(apiTorrents))
+	router.HandleFunc("/api/v0.1/torrentstotal", BasicAuth(apiTorrentsTotal))
 	router.HandleFunc("/api/v0.1/torrents/{infohash}", BasicAuth(infohashMiddleware(apiTorrent)))
 	router.HandleFunc("/api/v0.1/torrents/{infohash}/filelist", BasicAuth(infohashMiddleware(apiFileList)))
 

web/torrents.go

@@ -198,6 +198,51 @@ func apiTorrents(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func apiTorrentsTotal(w http.ResponseWriter, r *http.Request) {
+	// @lastOrderedValue AND @lastID are either both supplied or neither of them should be supplied
+	// at all; and if that is NOT the case, then return an error.
+	if q := r.URL.Query(); !((q.Get("lastOrderedValue") != "" && q.Get("lastID") != "") ||
+		(q.Get("lastOrderedValue") == "" && q.Get("lastID") == "")) {
+		http.Error(w, "`lastOrderedValue`, `lastID` must be supplied altogether, if supplied.", http.StatusBadRequest)
+		return
+	}
+
+	var tq struct {
+		Epoch int64  `schema:"epoch"`
+		Query string `schema:"query"`
+	}
+
+	err := r.ParseForm()
+	if err != nil {
+		http.Error(w, "error while parsing the URL: "+err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if r.Form.Has("epoch") {
+		tq.Epoch, err = strconv.ParseInt(r.Form.Get("epoch"), 10, 64)
+		if err != nil {
+			http.Error(w, "error while parsing the URL: "+err.Error(), http.StatusBadRequest)
+			return
+		}
+	} else {
+		http.Error(w, "lack required parameters while parsing the URL: `epoch`", http.StatusBadRequest)
+		return
+	}
+
+	tq.Query = r.Form.Get("query")
+
+	torrentsTotal, err := database.GetNumberOfQueryTorrents(tq.Query, tq.Epoch)
+	if err != nil {
+		http.Error(w, "GetNumberOfQueryTorrents: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set(ContentType, ContentTypeJson)
+	if err = json.NewEncoder(w).Encode(torrentsTotal); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
 func parseOrderBy(s string) (persistence.OrderingCriteria, error) {
 	switch s {
 	case "RELEVANCE":