Skip to content

Commit

Permalink
Merge pull request #5369 from snyk/CLI-415-binary-url
Browse files Browse the repository at this point in the history 
feat: change binary download url
  • Loading branch information
@sandor-trombitas
sandor-trombitas authored Aug 16, 2024
2 parents 959b5b1 + 89546b0 commit 438337e
Show file tree
Hide file tree
Showing 6 changed files with 135 additions and 30 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ __outputs__
/ts-binary-wrapper/README.md
/ts-binary-wrapper/SECURITY.md
/ts-binary-wrapper/src/generated
/ts-binary-wrapper/node_modules

# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
Expand Down
5 changes: 3 additions & 2 deletions ts-binary-wrapper/src/bootstrap.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@ const errorContextMessage = 'Download Error';

if (process.argv.includes('exec')) {
const filenameShasum = config.getShasumFile();
const downloadUrl = config.getDownloadLocation();
const { downloadUrl, backupUrl } = config.getDownloadLocations();

const downloadError = await common.downloadExecutable(
const downloadError = await common.downloadWithBackup(
downloadUrl,
backupUrl,
executable,
filenameShasum,
);
Expand Down
83 changes: 68 additions & 15 deletions ts-binary-wrapper/src/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const binaryDeploymentsFilePath = path.join(
'generated',
'binary-deployments.json',
);
export const integrationName = 'TS_BINARY_WRAPPER';

export class WrapperConfiguration {
private version: string;
Expand All @@ -45,9 +46,14 @@ export class WrapperConfiguration {
return this.binaryName;
}

public getDownloadLocation(): string {
const baseUrl = 'https://static.snyk.io/cli/v';
return baseUrl + this.version + '/' + this.binaryName;
public getDownloadLocations(): { downloadUrl: string; backupUrl: string } {
const baseUrl = 'https://downloads.snyk.io/cli';
const backupUrl = 'https://static.snyk.io/cli';

return {
downloadUrl: `${baseUrl}/v${this.version}/${this.binaryName}`,
backupUrl: `${backupUrl}/v${this.version}/${this.binaryName}`,
};
}

public getLocalLocation(): string {
Expand All @@ -60,6 +66,10 @@ export class WrapperConfiguration {
}
}

const logErrorWithTimeStamps = (...args) => {
console.error(`${new Date().toISOString()}:`, ...args);
};

export function determineBinaryName(platform: string, arch: string): string {
let osname = platform;
let archname = arch;
Expand Down Expand Up @@ -174,24 +184,33 @@ export function runWrapper(executable: string, cliArguments: string[]): number {
const debug = debugEnabled(cliArguments);

if (debug) {
console.error('Executing: ' + executable + ' ' + cliArguments.join(' '));
logErrorWithTimeStamps(
'Executing: ' + executable + ' ' + cliArguments.join(' '),
);
}

const res = spawnSync(executable, cliArguments, {
shell: false,
stdio: 'inherit',
env: {
...process.env,
SNYK_INTEGRATION_NAME: integrationName,
SNYK_INTEGRATION_VERSION: getCurrentVersion(versionFile),
},
});

if (res.status !== null) {
if (debug) {
console.error(res);
logErrorWithTimeStamps(res);
}

return res.status;
} else {
console.error(res);
logErrorWithTimeStamps(res);
if (!formatErrorMessage((res.error as SpawnError).code)) {
console.error('Failed to spawn child process. (' + executable + ')');
logErrorWithTimeStamps(
'Failed to spawn child process. (' + executable + ')',
);
}

return 2;
Expand Down Expand Up @@ -232,7 +251,7 @@ export function formatErrorMessage(message: string): boolean {
return false;
}

console.error(getWarningMessage(warning));
logErrorWithTimeStamps(getWarningMessage(warning));
return true;
}

Expand All @@ -242,7 +261,8 @@ export function downloadExecutable(
filenameShasum: string,
): Promise<Error | undefined> {
return new Promise<Error | undefined>(function(resolve) {
const options = new URL(downloadUrl);
logErrorWithTimeStamps('Starting download');
const options = new URL(`${downloadUrl}?utm_source=${integrationName}`);
const temp = path.join(__dirname, Date.now().toString());
const fileStream = fs.createWriteStream(temp);
const shasum = createHash('sha256').setEncoding('hex');
Expand Down Expand Up @@ -271,19 +291,19 @@ export function downloadExecutable(
if (filenameShasum && actualShasum != filenameShasum) {
cleanupAfterError(Error('Shasum comparison failed!\n' + debugMessage));
} else {
console.error(debugMessage);
logErrorWithTimeStamps(debugMessage);

// finally rename the file and change permissions
fs.renameSync(temp, filename);
fs.chmodSync(filename, 0o755);
console.error('Downloaded successfull! ');
logErrorWithTimeStamps('Downloaded successfull! ');
}

resolve(undefined);
});

console.error(
"Downloading from '" + downloadUrl + "' to '" + filename + "'",
logErrorWithTimeStamps(
"Downloading from '" + options.toString() + "' to '" + filename + "'",
);

const req = https.get(options, (res) => {
Expand Down Expand Up @@ -322,9 +342,42 @@ export function downloadExecutable(
});
}

export async function downloadWithBackup(
downloadUrl: string,
backupUrl: string,
filename: string,
filenameShasum: string,
): Promise<Error | undefined> {
try {
const error = await downloadExecutable(
downloadUrl,
filename,
filenameShasum,
);
if (error) {
logErrorWithTimeStamps(error);
logErrorWithTimeStamps(
`Failed to download from ${downloadUrl}! Trying to download from ${backupUrl} location...`,
);
const backupError = await downloadExecutable(
backupUrl,
filename,
filenameShasum,
);

logErrorWithTimeStamps(backupError);
return backupError;
}
} catch (err) {
// Handle any unexpected errors
logErrorWithTimeStamps('An unexpected error occurred:', err);
throw err; // Rethrow if you want to propagate the error upwards
}
}

export async function logError(
context: string,
err,
err: Error,
printToConsole = true,
): Promise<void> {
if (isAnalyticsEnabled()) {
Expand All @@ -345,7 +398,7 @@ export async function logError(

// finally log the error to the console as well
if (printToConsole) {
console.error('\n' + err);
logErrorWithTimeStamps('\n' + err);
formatErrorMessage(err.message);
}
}
Expand Down
6 changes: 4 additions & 2 deletions ts-binary-wrapper/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@ function run(executable: string): number {
try {
const config = common.getCurrentConfiguration();
const executable = config.getLocalLocation();
const { downloadUrl, backupUrl } = config.getDownloadLocations();

if (!fs.existsSync(executable)) {
console.error("Executable doesn't exist, trying to download.");

const downloadError = await common.downloadExecutable(
config.getDownloadLocation(),
const downloadError = await common.downloadWithBackup(
downloadUrl,
backupUrl,
executable,
config.getShasumFile(),
);
Expand Down
66 changes: 57 additions & 9 deletions ts-binary-wrapper/test/unit/common.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ describe('Get Shasum', () => {
describe('Configuration', () => {
it('Download and local location', async () => {
const expectedDownloadLocation =
'https://static.snyk.io/cli/v1.2.3/snyk-win.exe';
'https://downloads.snyk.io/cli/v1.2.3/snyk-win.exe';
const expectedLocalLocation = path.join(
__dirname,
'..',
Expand All @@ -146,7 +146,7 @@ describe('Configuration', () => {
'1234abcdef',
);

const actualDownloadLocation = config.getDownloadLocation();
const actualDownloadLocation = config.getDownloadLocations().downloadUrl;
expect(actualDownloadLocation).toEqual(expectedDownloadLocation);

const actualLocalLocation = config.getLocalLocation();
Expand Down Expand Up @@ -219,17 +219,18 @@ describe('Testing binary bootstrapper', () => {

// download the shasum first, here we don't expect a shasum comparison
const shasumDownload = await common.downloadExecutable(
config.getDownloadLocation() + shafileExtension,
config.getDownloadLocations().downloadUrl + shafileExtension,
shasumFile,
'',
);
expect(shasumDownload).toBeUndefined();
expect(fs.existsSync(shasumFile)).toBeTruthy();
const expectedShasum = common.getCurrentSha256sum(binaryName, shasumFile);

const { downloadUrl } = config.getDownloadLocations();
// download binary next and use previously downloaded shasum to check validity
const binaryDownload = await common.downloadExecutable(
config.getDownloadLocation(),
downloadUrl,
config.getLocalLocation(),
expectedShasum,
);
Expand All @@ -241,10 +242,55 @@ describe('Testing binary bootstrapper', () => {

try {
// check if the binary is executable
fs.accessSync(config.getLocalLocation(), fs.constants.X_OK);
expect(
fs.accessSync(config.getLocalLocation(), fs.constants.X_OK),
).not.toThrow();
} catch {
// execution of binary not possible
}

fs.unlinkSync(shasumFile);
fs.unlinkSync(config.getLocalLocation());
});
it('downloadWithBackup() succesfull', async () => {
const binaryName = 'snyk-macos';
const shafileExtension = '.sha256';
const config = new common.WrapperConfiguration('1.1080.0', binaryName, '');
const shasumFile =
config.getLocalLocation() + Math.random() + shafileExtension;
const { downloadUrl } = config.getDownloadLocations();

// download the shasum first, here we don't expect a shasum comparison
const shasumDownload = await common.downloadWithBackup(
'https://notdownloads.snyk.io/cli/v1.1080.0/snyk-macos.sha256',
downloadUrl + shafileExtension,
shasumFile,
'',
);
expect(shasumDownload).toBeUndefined();
expect(fs.existsSync(shasumFile)).toBeTruthy();
const expectedShasum = common.getCurrentSha256sum(binaryName, shasumFile);

// download binary next and use previously downloaded shasum to check validity
const binaryDownload = await common.downloadWithBackup(
'https://notdownloads.snyk.io/cli/v1.1080.0/snyk-macos',
downloadUrl,
config.getLocalLocation(),
expectedShasum,
);
expect(binaryDownload).toBeUndefined();
expect(fs.existsSync(config.getLocalLocation())).toBeTruthy();

const stats = fs.statSync(config.getLocalLocation());
expect(stats.mode).toEqual(0o100755);

try {
// check if the binary is executable
expect(
fs.accessSync(config.getLocalLocation(), fs.constants.X_OK),
).not.toThrow();
} catch {
// execution of binary not possible
expect(false).toBeTruthy();
}

fs.unlinkSync(shasumFile);
Expand All @@ -257,10 +303,11 @@ describe('Testing binary bootstrapper', () => {
const config = new common.WrapperConfiguration('1.1080.0', binaryName, '');
const shasumFile =
config.getLocalLocation() + Math.random() + shafileExtension;
const { downloadUrl } = config.getDownloadLocations();

// download just any file and state a shasum expectation that never can be fullfilled
const shasumDownload = await common.downloadExecutable(
config.getDownloadLocation() + shafileExtension,
downloadUrl + shafileExtension,
shasumFile,
'incorrect-shasum',
);
Expand All @@ -274,10 +321,11 @@ describe('Testing binary bootstrapper', () => {
const config = new common.WrapperConfiguration('1.1080.0', binaryName, '');
const shasumFile =
config.getLocalLocation() + Math.random() + shafileExtension;
const { downloadUrl } = config.getDownloadLocations();

// try to download a file that doesn't exis
const shasumDownload = await common.downloadExecutable(
config.getDownloadLocation() + shafileExtension,
downloadUrl + shafileExtension,
shasumFile,
'incorrect-shasum',
);
Expand All @@ -298,7 +346,7 @@ describe('Testing binary bootstrapper', () => {
});
});

describe('isAnalyticsEnabled ', () => {
describe('isAnalyticsEnabled', () => {
it('enabled', async () => {
delete process.env.SNYK_DISABLE_ANALYTICS;
expect(common.isAnalyticsEnabled()).toBeTruthy();
Expand Down
4 changes: 2 additions & 2 deletions ts-binary-wrapper/test/util/prepareEnvironment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,6 @@ export class TestEnvironmentSetup {
if (process.argv.includes('exec')) {
(async function() {
const env = new TestEnvironmentSetup();
await env.prepareEnvironment('1.1080.0');
});
await env.prepareEnvironment('1.1292.1');
})();
}

0 comments on commit 438337e

Please sign in to comment.