From a241ea3e8eaec2e70a2d4689af2dfb4f3a349785 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 29 Apr 2024 10:42:55 +0100 Subject: [PATCH] fix: switch to GitHub downloads (#45) Web3Storage doesn't have free usage any more so switch to downloading tarballs from GitHub release pages instead and verifying them against an expected CID. --- Makefile | 8 ++-- README.md | 5 +- package.json | 21 ++++----- scripts/update-versions.js | 36 +++++++++++++++ scripts/upload.js | 59 ------------------------ src/arches.js | 9 ++++ src/download.js | 93 ++++++++++++++++++++------------------ src/hash-file.js | 33 ++++++++++++++ src/versions.json | 62 +++++++++++++------------ test/download.spec.js | 9 +++- 10 files changed, 182 insertions(+), 153 deletions(-) create mode 100644 scripts/update-versions.js delete mode 100644 scripts/upload.js create mode 100644 src/arches.js create mode 100644 src/hash-file.js diff --git a/Makefile b/Makefile index b787575..c80a612 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ -COMMIT?=v0.5.0 +COMMIT?=v0.6.0 TARGETS=linux darwin win32 WORKDIR=bin -all: clean darwin linux win32 +all: clean darwin linux win32 versions clean: rm -rf *.tar.gz *.zip bin/p2pd-* bin/go-libp2p-daemon @@ -44,7 +44,7 @@ win32: zip p2pd-$(COMMIT)-$@-arm64.zip $(WORKDIR)/p2pd-win32-arm64.exe && \ zip p2pd-$(COMMIT)-$@-386.zip $(WORKDIR)/p2pd-win32-386.exe -upload: - node ./scripts/upload.js +versions: + node ./scripts/update-versions.js .PHONY: clean diff --git a/README.md b/README.md index 3376907..9c17092 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,7 @@ archive the binaries and upload them to web3.storage. ```console $ make all ``` -2. Upload new versions - ```console - $ make upload - ``` +2. Upload new versions to the GitHub release page 3. Open a PR to this repo with changes made to `src/versions.json` If anything goes wrong: diff --git a/package.json b/package.json index 9dcccd1..013fad1 100644 --- a/package.json +++ b/package.json @@ -74,28 +74,27 @@ }, "dependencies": { "blockstore-core": "^4.2.0", + "browser-readablestream-to-it": "^2.0.7", "cachedir": "^2.3.0", "delay": "^6.0.0", - "got": "^12.5.3", "gunzip-maybe": "^1.4.2", "ipfs-unixfs-importer": "^15.1.5", "it-last": "^3.0.2", - "multiformats": "^11.0.2", - "p-retry": "^5.1.2", - "pkg-conf": "^4.0.0", - "tar-fs": "^2.1.0", - "uint8arrays": "^4.0.3", + "it-to-buffer": "^4.0.7", + "multiformats": "^13.1.0", + "p-retry": "^6.2.0", + "package-config": "^5.0.0", + "tar-fs": "^3.0.6", + "uint8arrays": "^5.0.3", "unzip-stream": "^0.3.0" }, "devDependencies": { - "@types/got": "^9.6.12", "@types/gunzip-maybe": "^1.4.0", "@types/tar-fs": "^2.0.1", "@types/unzip-stream": "^0.3.1", - "aegir": "^39.0.9", - "execa": "^7.0.0", - "pre-commit": "^1.2.2", - "web3.storage": "^4.5.4" + "aegir": "^42.2.9", + "execa": "^8.0.1", + "pre-commit": "^1.2.2" }, "pre-commit": "restore-bin" } diff --git a/scripts/update-versions.js b/scripts/update-versions.js new file mode 100644 index 0000000..e2cf5fd --- /dev/null +++ b/scripts/update-versions.js @@ -0,0 +1,36 @@ +/* eslint-disable no-console */ +import fs from 'node:fs' +import { join, resolve } from 'node:path' +import * as url from 'node:url' +import { hashFile } from '../src/hash-file.js' +import { ARCHITECTURES } from '../src/arches.js' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) +const versionsPath = join(__dirname, '..', 'src', 'versions.json') + +const version = fs.readFileSync(join(__dirname, '..', 'Makefile'), { + encoding: 'utf8' +}) + .split('\n') + .map(line => line.trim()) + .filter(line => line.startsWith('COMMIT?=')) + .pop() + .replace('COMMIT?=', '') + +const versions = {} + +for (const arch of ARCHITECTURES) { + const filePath = resolve(join(__dirname, '..', `p2pd-${version}-${arch}.${arch.includes('win32') ? 'zip' : 'tar.gz'}`)) + const cid = await hashFile(filePath) + versions[arch] = cid.toString() +} + +const manifest = JSON.parse(fs.readFileSync(versionsPath, { + encoding: 'utf8' +})) + +manifest.versions[version] = versions + +fs.writeFileSync(versionsPath, JSON.stringify(manifest, null, 2), { + encoding: 'utf8' +}) diff --git a/scripts/upload.js b/scripts/upload.js deleted file mode 100644 index 1d274dc..0000000 --- a/scripts/upload.js +++ /dev/null @@ -1,59 +0,0 @@ -import { join } from 'node:path' -import * as url from 'node:url' -import fs from 'node:fs' -import { Web3Storage, getFilesFromPath } from 'web3.storage' -import { API_TOKEN } from './.config.js' - -const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) - -// Construct with token and endpoint -const client = new Web3Storage({ token: API_TOKEN }) - -const commit = process.env.COMMIT ?? 'v0.5.0' - -const tarballs = [ - `p2pd-${commit}-darwin.tar.gz`, - `p2pd-${commit}-linux-386.tar.gz`, - `p2pd-${commit}-linux-amd64.tar.gz`, - `p2pd-${commit}-linux-arm64.tar.gz`, - `p2pd-${commit}-win32-386.zip`, - `p2pd-${commit}-win32-amd64.zip`, - `p2pd-${commit}-win32-arm64.zip` -] - -const output = {} - -// Pack files into a CAR and send to web3.storage -const rootCid = await client.put(await getFilesFromPath(tarballs.map(tarball => join(__dirname, '..', tarball))), { - onStoredChunk: (size) => { - console.info('stored', size) - }, - onRootCidReady: (cid) => { - console.info('root', cid) - } -}) // Promise - -console.info('upload complete') -console.info('root cid', rootCid) - -// Fetch and verify files from web3.storage -const res = await client.get(rootCid) // Promise -const files = await res.files() // Promise - -for (const file of files) { - output[file.name.split(`${commit}-`)[1].split('.')[0]] = file.cid -} - -console.info('updating src/versions.json') - -const { versions } = JSON.parse(fs.readFileSync(join(__dirname, '..', 'src', 'versions.json'), { - encoding: 'utf-8' -})) - -versions[commit] = output - -fs.writeFileSync(join(__dirname, '..', 'src', 'versions.json'), JSON.stringify({ latest: commit, versions }, null, 2), { - encoding: 'utf-8' -}) - -console.info('done') diff --git a/src/arches.js b/src/arches.js new file mode 100644 index 0000000..1cc4c95 --- /dev/null +++ b/src/arches.js @@ -0,0 +1,9 @@ +export const ARCHITECTURES = [ + 'darwin', + 'linux-386', + 'linux-amd64', + 'linux-arm64', + 'win32-386', + 'win32-amd64', + 'win32-arm64' +] diff --git a/src/download.js b/src/download.js index f7a0bef..79ffa0b 100644 --- a/src/download.js +++ b/src/download.js @@ -12,22 +12,19 @@ import os from 'node:os' import path from 'node:path' import * as url from 'node:url' import util from 'node:util' -import { BlackHoleBlockstore } from 'blockstore-core/black-hole' +import browserReadableStreamToIt from 'browser-readablestream-to-it' import cachedir from 'cachedir' import delay from 'delay' -import got from 'got' import gunzip from 'gunzip-maybe' -import { importer } from 'ipfs-unixfs-importer' -import { fixedSize } from 'ipfs-unixfs-importer/chunker' -import { balanced } from 'ipfs-unixfs-importer/layout' -import last from 'it-last' +import toBuffer from 'it-to-buffer' import { CID } from 'multiformats/cid' import retry from 'p-retry' -import { packageConfigSync } from 'pkg-conf' +import { packageConfigSync } from 'package-config' import tarFS from 'tar-fs' import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import unzip from 'unzip-stream' import * as goenv from './go-platform.js' +import { hashFile } from './hash-file.js' const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) const isWin = process.platform === 'win32' @@ -36,6 +33,8 @@ const { latest, versions } = JSON.parse(fs.readFileSync(path.join(__dirname, 've encoding: 'utf-8' })) +const DOWNLOAD_TIMEOUT_MS = 60000 + /** * avoid expensive fetch if file is already in cache * @@ -63,8 +62,34 @@ async function cachingFetchAndVerify (url, cid, options = {}) { console.info(`Cached file ${cachedFilePath} not found`) console.info(`Downloading ${url} to ${cacheDir}`) - const buf = await retry(async (attempt) => { - return await got(url).buffer() + const buf = await retry(async () => { + const signal = AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) + + try { + const res = await fetch(url, { + signal + }) + + console.info(`${url} ${res.status} ${res.statusText}`) + + if (!res.ok) { + throw new Error(`${res.status}: ${res.statusText}`) + } + + const body = res.body + + if (body == null) { + throw new Error('Response had no body') + } + + return await toBuffer(browserReadableStreamToIt(body)) + } catch (err) { + if (signal.aborted) { + console.error(`Download timed out after ${DOWNLOAD_TIMEOUT_MS}ms`) + } + + throw err + } }, { retries, onFailedAttempt: async (err) => { @@ -82,39 +107,7 @@ async function cachingFetchAndVerify (url, cid, options = {}) { } console.info(`Verifying ${filename} from ${cachedFilePath}`) - - const blockstore = new BlackHoleBlockstore() - const input = fs.createReadStream(cachedFilePath) - let result - - if (cid.startsWith('bafy')) { - console.info('Recreating new-style CID') - // new-style w3storage CID - result = await last(importer([{ - content: input - }], blockstore, { - cidVersion: 1, - rawLeaves: true, - chunker: fixedSize({ chunkSize: 1024 * 1024 }), - layout: balanced({ maxChildrenPerNode: 1024 }) - })) - } else { - // old-style kubo CID - result = await last(importer([{ - content: input - }], blockstore, { - cidVersion: 0, - rawLeaves: false, - chunker: fixedSize({ chunkSize: 262144 }), - layout: balanced({ maxChildrenPerNode: 174 }) - })) - } - - if (result == null) { - throw new Error('Import failed') - } - - const receivedCid = result.cid + const receivedCid = await hashFile(cachedFilePath) const downloadedCid = CID.parse(cid) if (!uint8ArrayEquals(downloadedCid.multihash.bytes, receivedCid.multihash.bytes)) { @@ -168,7 +161,7 @@ function cleanArguments (options = {}) { cwd: process.env.INIT_CWD ?? process.cwd(), defaults: { version: options.version ?? latest, - distUrl: 'https://%s.ipfs.w3s.link' + distUrl: 'https://github.com/libp2p/go-libp2p-daemon/releases/download/%s/p2pd-%s-%s.%s' } }) @@ -206,8 +199,20 @@ async function getDownloadURL (version, platform, arch, distUrl) { throw new Error(`No binary available for platform '${platform}' and/or arch ${arch}`) } + let downloadTarget = `${platform}-${arch}` + + if (platform === 'darwin') { + downloadTarget = 'darwin' + } + + let extension = 'tar.gz' + + if (platform === 'win32') { + extension = 'zip' + } + return { - url: util.format(distUrl, CID.parse(cid).toV1().toString()), + url: util.format(distUrl, version, version, downloadTarget, extension), cid } } diff --git a/src/hash-file.js b/src/hash-file.js new file mode 100644 index 0000000..57a7ab4 --- /dev/null +++ b/src/hash-file.js @@ -0,0 +1,33 @@ +import fs from 'node:fs' +import { BlackHoleBlockstore } from 'blockstore-core/black-hole' +import { importer } from 'ipfs-unixfs-importer' +import { fixedSize } from 'ipfs-unixfs-importer/chunker' +import { balanced } from 'ipfs-unixfs-importer/layout' +import last from 'it-last' + +/** + * @typedef {import('multiformats/cid').CID} CID + */ + +/** + * @param {string} filePath + * @returns {Promise} + */ +export async function hashFile (filePath) { + const blockstore = new BlackHoleBlockstore() + const input = fs.createReadStream(filePath) + const result = await last(importer([{ + content: input + }], blockstore, { + cidVersion: 1, + rawLeaves: true, + chunker: fixedSize({ chunkSize: 1024 * 1024 }), + layout: balanced({ maxChildrenPerNode: 1024 }) + })) + + if (result == null) { + throw new Error('Import failed') + } + + return result.cid +} diff --git a/src/versions.json b/src/versions.json index 39ea860..abc0fd6 100644 --- a/src/versions.json +++ b/src/versions.json @@ -1,39 +1,41 @@ { - "latest": "v0.5.0", + "latest": "v0.6.0", "versions": { + "v0.6.0": { + "darwin": "bafybeieplo3tqhpdfptzvflxbdlvw32vivyq7qkcukytby2iwjuhxwl3dy", + "linux-386": "bafybeibexjcu5cznafpkzdhqy4kr4yvuzta4fbl23pvpafyghpenpljyye", + "linux-amd64": "bafybeicvwncyy4djmluzpijhcytkzucwq6u3fiwfj67h3s35zgiuqwxouy", + "linux-arm64": "bafybeihisjdyw27jvomzt5lh4up74y6us2ty427re746mfm56ktl4xe25i", + "win32-386": "bafybeidqsj2v2vs6cy2rpdvgvppgiade75ixk5xhrskfk633sz7p2e5hu4", + "win32-amd64": "bafybeigpicf45xmla45jrhv6h7y7u5wm6ceumsv4uh6a24yrdsdjm5j3w4", + "win32-arm64": "bafybeiagcakk7xmp4igcird6wltezi5cmgg7hy24zmw4gvsbhrvbpqqfbq" + }, "v0.5.0": { - "darwin": "bafybeievsa472emtpzcl5yjykw2mnqq7xav2aisk4rxrokhhtda6s3puau", - "linux-386": "bafybeicbnoxi3lboombgqjxhjniilwclzc3vv52dmcmppian5dummflygq", - "linux-amd64": "bafybeibhzp5scss4rgo7cnxvse2fnoqso52wef3hmbtqfv6eyx7munrcc4", - "linux-arm64": "bafybeicaae3vrczyifdzi4kcad7isijujqgu4plupii3rbkluq77xf3hiu", - "win32-386": "bafybeihkrynwltb5mva5m33jxynww5ih7xzwh23xtq4qtk55zdc2gxgp7i", - "win32-amd64": "bafybeicpm2jp3qsx2uh4gibwz4vfbil3nuxddlsiz4kwc3e3ghkof3pbty", - "win32-arm64": "bafybeigkro23bzxeadltjk46zc47y3sgs5nlckwmf64pnrqvjs4oc5at3y" + "darwin": "bafybeifcyp5oxna7futy4xsmun46s7fgwdobchflvfwet7m5ygzsj4leem", + "linux-386": "bafybeihwf4uhtb4lqlap76weon7dffsnwzjrgfcru6chmxe3wa72fl3rkq", + "linux-amd64": "bafybeiarqwbdey3qsv6xk7i6pidgjyilfiootl5ue6vchniirtnlealu6i", + "linux-arm64": "bafybeidav3iude4rttb3pd2zolzuqudc3lqrjjsoyfpmkyzz4y44mkqaxu", + "win32-386": "bafybeibiktl5cab6vlirnrcsbt474vo3yg7i4vexagdjtwzs76kh6oyq3e", + "win32-amd64": "bafybeifs6xvbdczsacwprg6qsdfihmyso4ghciwwvilrefdvf3otyksfrq", + "win32-arm64": "bafybeihjp3byllb5eqjettg5kdiagvzm5jdhfi5qa26b33cbbrdrw337nm" }, "v0.4.1": { - "darwin": "QmRFbrfLm4DWDmZHgubAVqXshRaV5vSbzhM5D7kAmZrdug", - "linux": "QmSMiiCCmu4YGdDfJL8cpbEeDtUGUULnvHtTBCeio8zbe7", - "win32": "QmYrYKthxvoFZnLFP1hJX7BFTvQgoSip3mwNNuaKDZ3VTU" + "darwin": "bafybeie3zgxb4bgoucchwwkjuvq25ioulns4dqwkwtnz3mzqkbszdytjnm", + "linux-386": "bafybeiam3cqcjm44rongvpgouqttvvygrs3jq4phqadwyhx4l5updapc3e", + "linux-amd64": "bafybeibyhq5qvi2x24zwztsghh63c3xwi3vtpzy5gbo6mxfa4353ofwrqm", + "linux-arm64": "bafybeidik6zb4xpsyuwzohmz252abi54fodfgirwbkvoshv2i4zvi6pzma", + "win32-386": "bafybeibbzbt2ioxzlp66ebkluyk5ekxqxuwwuci4o5cswwjw3x4etujeau", + "win32-amd64": "bafybeidtqdbx34wsfkyvszdjrl4fzayhesuzu4vn67mghqbi7yonlqzsuq", + "win32-arm64": "bafybeiajwj7ulvyjpw535fxtjhkrdadvjk2cjkwrrkrxjyd6l5qbmcfj64" }, "v0.4.0": { - "darwin": "QmXTAqBwpuAyYTsWD7gZ1MBq77cWmkeh1Lm8e286P47C7C", - "linux": "QmZwvFy5e3RrxD39DcXLGHfhsga8uMEU3JpKKVbHfkcCdv", - "win32": "QmdCyyCxLgAD3hNLAgYDRioKufPkyr5ggGnPokkW9FcfcZ" - }, - "v0.3.1": { - "darwin": "QmettTVLnaLaVF9DzKd2yik8fnGZ1rbK7PuHBF8eqB2Yft", - "linux": "QmQdVisPPnESnvQYmify1AEULwG5puXQq1c6yQyD4AVCJd", - "win32": "QmesyDksxpWCMHsKHM371bwhh3J7kmg9HrdqYg7WfH6TS2" - }, - "v0.1.0": { - "darwin": "QmajN9chpFzG4msziqEKRbF32VW87a5SeKRKXiao6qokVT", - "linux": "QmaX7i8cFkVoN8FmgLhNaxfapkQrztpuChdMRgcYDZQmq1", - "win32": "Qme6SrCD6fm8AFmH9kA5BYwyz3fBq2QjrsxePen5Xoy9R1" - }, - "v6.0.30": { - "darwin": "QmSS6iQ3JNJ96g73tFARAcdYaMmAsvjX1RVuMPqJThTs7D", - "linux": "Qmdu8WCFUW43u8mHpbj2jV2ZSB1mP7JcjnJJcJ1oF6iMwW", - "win32": "QmP5P6AYQjbG26j5qrzbohSqAW3AP2Mvsuun3fS99hC1x6" + "darwin": "bafybeiafw7yn3c2uqu4wotihi6q7n2ycxt6c37pj2jcflbfdwpijmlgb2m", + "linux-386": "bafybeibsssm27rgbop3eriigeeqhxikilr65xygqll27sgiaaiz3jinaa4", + "linux-amd64": "bafybeicasjgbsjwflr23obayqg2rmt5m26j7jggy4slx26mafagsyhxv6a", + "linux-arm64": "bafybeiasanseziia3u5tp75zyqbtz3rxzwhvkwtsdwm6o7vhu2j77dy52m", + "win32-386": "bafybeia6e7b64orubdl23m5mqlnllemyk65ncoc3xqm3jhoky6acnpz3jm", + "win32-amd64": "bafybeidf2gpwbe6uss7eotncqbctzxlnj4rdplyjh5ljbcjwmp2nuvxgxa", + "win32-arm64": "bafybeid6agdkcb2k7hpm2x6v3ndxrshh7qogr4c2tvn2mit7rvnwy6sopm" } } -} \ No newline at end of file +} diff --git a/test/download.spec.js b/test/download.spec.js index e0a82d9..2717164 100644 --- a/test/download.spec.js +++ b/test/download.spec.js @@ -1,6 +1,8 @@ /* eslint-env mocha */ import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' import { expect } from 'aegir/chai' import { download } from '../src/download.js' import { path as detectLocation } from '../src/index.js' @@ -8,9 +10,14 @@ import { clean } from './fixtures/clean.js' describe('download', () => { beforeEach(async () => { + process.env.NPM_GO_LIBP2P_CACHE = path.join(os.tmpdir(), `npm-go-libp2p-test-cache-${Date.now()}`) await clean() }) + afterEach(async () => { + delete process.env.NPM_GO_LIBP2P_CACHE + }) + it('downloads libp2p (current version and platform)', async () => { const installPath = await download() const stats = await fs.stat(installPath) @@ -27,7 +34,7 @@ describe('download', () => { it('returns an error when dist url is 404', async () => { process.env.GO_LIBP2P_DIST_URL = 'https://dist.ipfs.io/notfound' - await expect(download({ version: 'v0.3.1', retries: 0 })).to.eventually.be.rejected + await expect(download({ version: 'v0.4.0', retries: 0 })).to.eventually.be.rejected .with.property('message').that.matches(/404/) delete process.env.GO_LIBP2P_DIST_URL