Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Search paging #596

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

WIP: Search paging #596

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3bda28a
Added paged results to search tests.
j-rewerts Oct 22, 2019
fb4cc02
Assert the exact response length of all search tests.
j-rewerts Oct 22, 2019
cd8c27f
Fixed error handling.
j-rewerts Oct 22, 2019
5718843
Fixed duplicate params when paging.
j-rewerts Oct 22, 2019
8b54caf
Added single page search.
j-rewerts Oct 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions lib/Requestable.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ class Requestable {
}
results.push(...thisGroup);

const nextUrl = getNextPage(response.headers.link);
let nextUrl = getNextPage(response.headers.link);
if(nextUrl) {
if (!options) {
options = {};
Expand All @@ -266,6 +266,8 @@ class Requestable {
.split('=')
.pop()
);
// Strip out query string, as we'll use 'options'.
nextUrl = nextUrl.split('?', 1)[0];
if (!(options && typeof options.page !== 'number')) {
log(`getting next page: ${nextUrl}`);
return this._requestAllPages(nextUrl, options, cb, results);
Expand All @@ -280,6 +282,44 @@ class Requestable {
return response;
}).catch(callbackErrorOrThrow(cb, path));
}

/**
* Make a request and fetch a specific page. Use this to override the paging provided
* by {@link Requestable#requestAllPages}.
* @param {string} path - the path to request
* @param {string} perPage - the maximum number of results to return
* @param {string} page - the page number
* @param {Object} options - the query parameters to include. It doesn't matter if this includes 'per_page' or
* 'page'.
* @param {Requestable.callback} [cb] - the function to receive the data. The returned data will always be an array.
* @return {Promise} - a promise which will resolve the page has been fetched
*/
_requestPage(path, perPage, page, options, cb) {
options['page'] = page;
options['per_page'] = perPage;

let results = [];
return this._request('GET', path, options)
.then((response) => {
let thisGroup;
if (response.data instanceof Array) {
thisGroup = response.data;
} else if (response.data.items instanceof Array) {
thisGroup = response.data.items;
} else {
let message = `cannot figure out how to append ${response.data} to the result set`;
throw new ResponseError(message, path, response);
}
results.push(...thisGroup);

if (cb) {
cb(null, results, response);
}

response.data = results;
return response;
}).catch(callbackErrorOrThrow(cb, path));
}
}

module.exports = Requestable;
Expand All @@ -306,7 +346,7 @@ function getNextPage(linksHeader = '') {
function callbackErrorOrThrow(cb, path) {
return function handler(object) {
let error;
if (object.hasOwnProperty('config')) {
if (object.hasOwnProperty('config') && object.response) {
const {response: {status, statusText}, config: {method, url}} = object;
let message = (`${status} error making request ${method} ${url}: "${statusText}"`);
error = new ResponseError(message, path, object);
Expand Down
6 changes: 5 additions & 1 deletion lib/Search.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,11 @@ class Search extends Requestable {
});

log(`searching ${path} with options:`, requestOptions);
return this._requestAllPages(`/search/${path}`, requestOptions, cb);
if (requestOptions.page && requestOptions.per_page) {
return this._requestPage(`/search/${path}`, requestOptions.per_page, requestOptions.page, requestOptions, cb);
} else {
return this._requestAllPages(`/search/${path}`, requestOptions, cb);
}
}

/**
Expand Down
28 changes: 14 additions & 14 deletions test/fixtures/search.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8911,7 +8911,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:11 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -8939,7 +8939,7 @@
{
"scope": "https://api.github.com:443",
"method": "GET",
"path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=2&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100",
"path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=2",
"body": "",
"status": 200,
"response": {
Expand Down Expand Up @@ -17848,7 +17848,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:14 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -17876,7 +17876,7 @@
{
"scope": "https://api.github.com:443",
"method": "GET",
"path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=3&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100",
"path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=3",
"body": "",
"status": 200,
"response": {
Expand Down Expand Up @@ -26785,7 +26785,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:17 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -26813,7 +26813,7 @@
{
"scope": "https://api.github.com:443",
"method": "GET",
"path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=4&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100",
"path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=4",
"body": "",
"status": 200,
"response": {
Expand Down Expand Up @@ -35722,7 +35722,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:20 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -35750,7 +35750,7 @@
{
"scope": "https://api.github.com:443",
"method": "GET",
"path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=5&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100",
"path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=5",
"body": "",
"status": 200,
"response": {
Expand Down Expand Up @@ -42256,7 +42256,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:22 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -42885,7 +42885,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:23 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -48918,7 +48918,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:24 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -48946,7 +48946,7 @@
{
"scope": "https://api.github.com:443",
"method": "GET",
"path": "/search/issues?q=windows+pip+label%3Abug+language%3Apython+state%3Aopen+&sort=created&order=asc&type=all&per_page=100&page=2&q=windows+pip+label:bug+language:python+state:open+&sort=created&order=asc&type=all&per_page=100",
"path": "/search/issues?q=windows+pip+label:bug+language:python+state:open+&sort=created&order=asc&type=all&per_page=100&page=2",
"body": "",
"status": 200,
"response": {
Expand Down Expand Up @@ -53447,7 +53447,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:26 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down Expand Up @@ -53564,7 +53564,7 @@
}
]
},
"headers": {
"rawHeaders": {
"server": "GitHub.com",
"date": "Wed, 22 Jun 2016 02:15:27 GMT",
"content-type": "application/json; charset=utf-8",
Expand Down
24 changes: 24 additions & 0 deletions test/search.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,27 @@ describe('Search', function() {
.then(function({data}) {
expect(data).to.be.an.array();
expect(data.length).to.be.above(0);
expect(data.length).to.be.equal(473);
});
});

it('should search repositories for specific page', function() {
let options;
let search = github.search({
q: 'tetris language:assembly',
sort: 'stars',
order: 'desc',
type: 'all',
per_page: '100',
page: '2',
});

return search.forRepositories(options)
.then(function({data}) {
expect(data).to.be.an.array();
expect(data.length).to.be.above(0);
expect(data.length).to.be.equal(100);
expect(data[0].name).to.be.equal('nand2tetris');
});
});

Expand All @@ -42,6 +63,7 @@ describe('Search', function() {
.then(function({data}) {
expect(data).to.be.an.array();
expect(data.length).to.be.above(0);
expect(data.length).to.be.equal(8);
});
});

Expand All @@ -57,6 +79,7 @@ describe('Search', function() {
.then(function({data}) {
expect(data).to.be.an.array();
expect(data.length).to.be.above(0);
expect(data.length).to.be.equal(161);
});
});

Expand All @@ -70,6 +93,7 @@ describe('Search', function() {
.then(function({data}) {
expect(data).to.be.an.array();
expect(data.length).to.be.above(0);
expect(data.length).to.be.equal(4);
});
});

Expand Down