From 3f765c5a4ad839b831fc66485f0086a8a900aebe Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 7 Oct 2024 12:56:59 -0400 Subject: [PATCH 01/27] move performance test for prepareRepo to integ --- .../amazonqFeatureDev/prepareRepoData.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename packages/core/src/{test => testInteg}/amazonqFeatureDev/prepareRepoData.test.ts (97%) diff --git a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts b/packages/core/src/testInteg/amazonqFeatureDev/prepareRepoData.test.ts similarity index 97% rename from packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts rename to packages/core/src/testInteg/amazonqFeatureDev/prepareRepoData.test.ts index 50a4093ceca..3e10a1bec42 100644 --- a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts +++ b/packages/core/src/testInteg/amazonqFeatureDev/prepareRepoData.test.ts @@ -5,7 +5,7 @@ import assert from 'assert' import { WorkspaceFolder } from 'vscode' import { performanceTest } from '../../shared/performance/performance' -import { createTestWorkspace } from '../testUtil' +import { createTestWorkspace } from '../../test/testUtil' import { prepareRepoData, TelemetryHelper } from '../../amazonqFeatureDev' import { AmazonqCreateUpload, getRandomString } from '../../shared' import { Span } from '../../shared/telemetry' From 5a0357ad0c3273269b6f8e734d9298d4236adf06 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 7 Oct 2024 13:02:09 -0400 Subject: [PATCH 02/27] move security scan test --- .../codewhisperer}/startSecurityScan.test.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) rename packages/core/src/{test/codewhisperer/commands => testInteg/codewhisperer}/startSecurityScan.test.ts (94%) diff --git a/packages/core/src/test/codewhisperer/commands/startSecurityScan.test.ts b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts similarity index 94% rename from packages/core/src/test/codewhisperer/commands/startSecurityScan.test.ts rename to packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts index f3efda3e040..b6bab378435 100644 --- a/packages/core/src/test/codewhisperer/commands/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts @@ -7,12 +7,12 @@ import assert from 'assert' import * as vscode from 'vscode' import * as sinon from 'sinon' import * as semver from 'semver' -import { DefaultCodeWhispererClient } from '../../../codewhisperer/client/codewhisperer' -import * as startSecurityScan from '../../../codewhisperer/commands/startSecurityScan' -import { SecurityPanelViewProvider } from '../../../codewhisperer/views/securityPanelViewProvider' -import { FakeExtensionContext } from '../../fakeExtensionContext' -import * as diagnosticsProvider from '../../../codewhisperer/service/diagnosticsProvider' -import { getTestWorkspaceFolder } from '../../../testInteg/integrationTestsUtilities' +import { DefaultCodeWhispererClient } from '../../codewhisperer' +import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' +import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' +import { FakeExtensionContext } from '../../test/fakeExtensionContext' +import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' +import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' import { join } from 'path' import { assertTelemetry, @@ -20,23 +20,23 @@ import { createTestWorkspaceFolder, getFetchStubWithResponse, toFile, -} from '../../testUtil' -import { stub } from '../../utilities/stubber' +} from '../../test/testUtil' +import { stub } from '../../test/utilities/stubber' import { AWSError, HttpResponse } from 'aws-sdk' -import { getTestWindow } from '../../shared/vscode/window' -import { SeverityLevel } from '../../shared/vscode/message' -import { cancel } from '../../../shared/localizedText' +import { getTestWindow } from '../../test/shared/vscode/window' +import { SeverityLevel } from '../../test/shared/vscode/message' +import { cancel } from '../../shared' import { showScannedFilesMessage, stopScanMessage, CodeAnalysisScope, projectScansLimitReached, -} from '../../../codewhisperer/models/constants' -import * as model from '../../../codewhisperer/models/model' -import { CodewhispererSecurityScan } from '../../../shared/telemetry/telemetry.gen' -import * as errors from '../../../shared/errors' -import * as timeoutUtils from '../../../shared/utilities/timeoutUtils' -import { performanceTest } from '../../../shared/performance/performance' +} from '../../codewhisperer' +import * as model from '../../codewhisperer/models/model' +import { CodewhispererSecurityScan } from '../../shared/telemetry/telemetry.gen' +import * as errors from '../../shared/errors' +import * as timeoutUtils from '../../shared/utilities/timeoutUtils' +import { performanceTest } from '../../shared/performance/performance' const mockCreateCodeScanResponse = { $response: { From e07593f89a5ce021ab069d9241af32d6ddbcd378 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 09:12:29 -0400 Subject: [PATCH 03/27] split up perf and non perf tests --- .../startSecurityScan.test.ts | 446 ++++++++++++++++++ .../codewhisperer/startSecurityScan.test.ts | 352 +------------- 2 files changed, 457 insertions(+), 341 deletions(-) create mode 100644 packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts diff --git a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts new file mode 100644 index 00000000000..7f668398992 --- /dev/null +++ b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts @@ -0,0 +1,446 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import * as vscode from 'vscode' +import * as sinon from 'sinon' +import * as assert from 'assert' +import * as semver from 'semver' +import * as model from '../../codewhisperer/models/model' +import * as timeoutUtils from '../../shared/utilities/timeoutUtils' +import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' +import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' +import * as errors from '../../shared/errors' +import { + SecurityPanelViewProvider, + stopScanMessage, + showScannedFilesMessage, + projectScansLimitReached, +} from '../../codewhisperer' +import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' +import { FakeExtensionContext } from '../fakeExtensionContext' +import { join } from 'path' +import { assertTelemetry, closeAllEditors, getFetchStubWithResponse } from '../testUtil' +import { stub } from '../../test/utilities/stubber' +import { DefaultCodeWhispererClient } from '../../codewhisperer' +import { AWSError, HttpResponse } from 'aws-sdk' +import { CodeAnalysisScope } from '../../codewhisperer' +import { getTestWindow } from '../shared/vscode/window' +import { SeverityLevel } from '../../test/shared/vscode/message' +import { cancel, CodewhispererSecurityScan } from '../../shared' + +const mockCreateCodeScanResponse = { + $response: { + data: { + jobId: 'jobId', + status: 'Pending', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + jobId: 'jobId', + status: 'Pending', +} + +const mockCreateUploadUrlResponse = { + $response: { + data: { + uploadId: 'uploadId', + uploadUrl: 'uploadUrl', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + uploadId: 'uploadId', + uploadUrl: 'https://test.com', +} + +const mockGetCodeScanResponse = { + $response: { + data: { + status: 'Completed', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + status: 'Completed', +} + +const mockCodeScanFindings = JSON.stringify([ + { + filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', + startLine: 1, + endLine: 1, + title: 'title', + description: { + text: 'text', + markdown: 'markdown', + }, + detectorId: 'detectorId', + detectorName: 'detectorName', + findingId: 'findingId', + relatedVulnerabilities: [], + severity: 'High', + remediation: { + recommendation: { + text: 'text', + url: 'url', + }, + suggestedFixes: [], + }, + codeSnippet: [], + } satisfies model.RawCodeScanIssue, +]) + +const mockListCodeScanFindingsResponse = { + $response: { + data: { + codeScanFindings: mockCodeScanFindings, + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + codeScanFindings: mockCodeScanFindings, +} + +describe('startSecurityScan', function () { + let extensionContext: FakeExtensionContext + let mockSecurityPanelViewProvider: SecurityPanelViewProvider + let appRoot: string + let appCodePath: string + let editor: vscode.TextEditor + const workspaceFolder = getTestWorkspaceFolder() + beforeEach(async function () { + extensionContext = await FakeExtensionContext.create() + mockSecurityPanelViewProvider = new SecurityPanelViewProvider(extensionContext) + appRoot = join(workspaceFolder, 'python3.7-plain-sam-app') + appCodePath = join(appRoot, 'hello_world', 'app.py') + editor = await openTestFile(appCodePath) + await model.CodeScansState.instance.setScansEnabled(false) + sinon.stub(timeoutUtils, 'sleep') + }) + afterEach(function () { + sinon.restore() + }) + after(async function () { + await closeAllEditors() + }) + const createClient = () => { + const mockClient = stub(DefaultCodeWhispererClient) + + mockClient.createCodeScan.resolves(mockCreateCodeScanResponse) + mockClient.createUploadUrl.resolves(mockCreateUploadUrlResponse) + mockClient.getCodeScan.resolves(mockGetCodeScanResponse) + mockClient.listCodeScanFindings.resolves(mockListCodeScanFindingsResponse) + return mockClient + } + + const openTestFile = async (filePath: string) => { + const doc = await vscode.workspace.openTextDocument(filePath) + return await vscode.window.showTextDocument(doc, { + selection: new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)), + }) + } + + it('Should render security scan result', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const commandSpy = sinon.spy(vscode.commands, 'executeCommand') + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + assert.ok(commandSpy.calledWith('workbench.action.problems.focus')) + assert.ok(securityScanRenderSpy.calledOnce) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + }) + + it('Should not focus problems panel for file scans', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const commandSpy = sinon.spy(vscode.commands, 'executeCommand') + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + + await model.CodeScansState.instance.setScansEnabled(true) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + assert.ok(commandSpy.neverCalledWith('workbench.action.problems.focus')) + assert.ok(securityScanRenderSpy.calledOnce) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'FILE', + passive: true, + }) + }) + + it('Should stop security scan for project scans when confirmed', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') + const testWindow = getTestWindow() + testWindow.onDidShowMessage((message) => { + if (message.message === stopScanMessage) { + message.selectItem(startSecurityScan.stopScanButton) + } + }) + model.codeScanState.setToRunning() + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + await startSecurityScan.confirmStopSecurityScan() + await scanPromise + assert.ok(securityScanRenderSpy.notCalled) + assert.ok(securityScanStoppedErrorSpy.calledOnce) + const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) + }) + + it('Should not stop security scan for project scans when not confirmed', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') + const testWindow = getTestWindow() + testWindow.onDidShowMessage((message) => { + if (message.message === stopScanMessage) { + message.selectItem(cancel) + } + }) + model.codeScanState.setToRunning() + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + await startSecurityScan.confirmStopSecurityScan() + await scanPromise + assert.ok(securityScanRenderSpy.calledOnce) + assert.ok(securityScanStoppedErrorSpy.notCalled) + const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) + }) + + it('Should stop security scan for file scans if setting is disabled', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') + await model.CodeScansState.instance.setScansEnabled(true) + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await model.CodeScansState.instance.setScansEnabled(false) + await scanPromise + assert.ok(securityScanRenderSpy.notCalled) + assert.ok(securityScanStoppedErrorSpy.calledOnce) + }) + + it('Should highlight files after scan is completed', async function () { + if (semver.lt(vscode.version, '1.78.0')) { + this.skip() + } + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const testWindow = getTestWindow() + testWindow.onDidShowMessage((message) => { + if (message.message.includes('Security scan completed')) { + message.selectItem(showScannedFilesMessage) + } + }) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanTotalIssues: 1, + codewhispererCodeScanIssuesWithFixes: 0, + codewhispererCodeScanScope: 'PROJECT', + passive: false, + } as CodewhispererSecurityScan) + }) + + it('Should cancel a scan if a newer one has started', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + await model.CodeScansState.instance.setScansEnabled(true) + + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await scanPromise + assertTelemetry('codewhisperer_securityScan', [ + { + result: 'Cancelled', + reasonDesc: 'Security scan stopped by user.', + reason: 'DefaultError', + } as unknown as CodewhispererSecurityScan, + { + result: 'Succeeded', + }, + ]) + }) + + it('Should not cancel a project scan if a file scan has started', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + await model.CodeScansState.instance.setScansEnabled(true) + + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await scanPromise + assertTelemetry('codewhisperer_securityScan', [ + { + result: 'Succeeded', + codewhispererCodeScanScope: 'FILE', + }, + { + result: 'Succeeded', + codewhispererCodeScanScope: 'PROJECT', + }, + ]) + }) + + it('Should handle failed scan job status', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + + const mockClient = createClient() + mockClient.getCodeScan.resolves({ + ...mockGetCodeScanResponse, + status: 'Failed', + }) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + mockClient, + extensionContext, + CodeAnalysisScope.PROJECT + ) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'PROJECT', + result: 'Failed', + reason: 'CodeScanJobFailedError', + reasonDesc: 'CodeScanJobFailedError: Security scan failed.', + passive: false, + } as unknown as CodewhispererSecurityScan) + }) + + it('Should show notification when throttled for project scans', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const mockClient = createClient() + mockClient.createCodeScan.throws({ + code: 'ThrottlingException', + time: new Date(), + name: 'error name', + message: 'Maximum project scan count reached for this month.', + } satisfies AWSError) + sinon.stub(errors, 'isAwsError').returns(true) + const testWindow = getTestWindow() + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + mockClient, + extensionContext, + CodeAnalysisScope.PROJECT + ) + assert.ok(testWindow.shownMessages.map((m) => m.message).includes(projectScansLimitReached)) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'PROJECT', + result: 'Failed', + reason: 'ThrottlingException', + reasonDesc: 'ThrottlingException: Maximum project scan count reached for this month.', + passive: false, + } as unknown as CodewhispererSecurityScan) + }) + + it('Should set monthly quota exceeded when throttled for file scans', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + await model.CodeScansState.instance.setScansEnabled(true) + const mockClient = createClient() + mockClient.createCodeScan.throws({ + code: 'ThrottlingException', + time: new Date(), + name: 'error name', + message: 'Maximum auto-scans count reached for this month.', + } satisfies AWSError) + sinon.stub(errors, 'isAwsError').returns(true) + assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), false) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + mockClient, + extensionContext, + CodeAnalysisScope.FILE + ) + assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), true) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'FILE', + result: 'Failed', + reason: 'ThrottlingException', + reasonDesc: 'ThrottlingException: Maximum auto-scans count reached for this month.', + passive: true, + } as unknown as CodewhispererSecurityScan) + }) +}) diff --git a/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts index b6bab378435..c07cce545b3 100644 --- a/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts @@ -2,17 +2,16 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ - -import assert from 'assert' import * as vscode from 'vscode' import * as sinon from 'sinon' -import * as semver from 'semver' -import { DefaultCodeWhispererClient } from '../../codewhisperer' import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' +import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' +import * as model from '../../codewhisperer/models/model' +import * as timeoutUtils from '../../shared/utilities/timeoutUtils' +import assert from 'assert' +import { DefaultCodeWhispererClient } from '../../codewhisperer' import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' import { FakeExtensionContext } from '../../test/fakeExtensionContext' -import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' -import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' import { join } from 'path' import { assertTelemetry, @@ -22,20 +21,10 @@ import { toFile, } from '../../test/testUtil' import { stub } from '../../test/utilities/stubber' -import { AWSError, HttpResponse } from 'aws-sdk' +import { HttpResponse } from 'aws-sdk' import { getTestWindow } from '../../test/shared/vscode/window' import { SeverityLevel } from '../../test/shared/vscode/message' -import { cancel } from '../../shared' -import { - showScannedFilesMessage, - stopScanMessage, - CodeAnalysisScope, - projectScansLimitReached, -} from '../../codewhisperer' -import * as model from '../../codewhisperer/models/model' -import { CodewhispererSecurityScan } from '../../shared/telemetry/telemetry.gen' -import * as errors from '../../shared/errors' -import * as timeoutUtils from '../../shared/utilities/timeoutUtils' +import { CodeAnalysisScope } from '../../codewhisperer' import { performanceTest } from '../../shared/performance/performance' const mockCreateCodeScanResponse = { @@ -132,330 +121,11 @@ const mockListCodeScanFindingsResponse = { codeScanFindings: mockCodeScanFindings, } -let extensionContext: FakeExtensionContext -let mockSecurityPanelViewProvider: SecurityPanelViewProvider -let appRoot: string -let appCodePath: string -let editor: vscode.TextEditor - -describe('startSecurityScan', function () { - const workspaceFolder = getTestWorkspaceFolder() - beforeEach(async function () { - extensionContext = await FakeExtensionContext.create() - mockSecurityPanelViewProvider = new SecurityPanelViewProvider(extensionContext) - appRoot = join(workspaceFolder, 'python3.7-plain-sam-app') - appCodePath = join(appRoot, 'hello_world', 'app.py') - editor = await openTestFile(appCodePath) - await model.CodeScansState.instance.setScansEnabled(false) - sinon.stub(timeoutUtils, 'sleep') - }) - afterEach(function () { - sinon.restore() - }) - after(async function () { - await closeAllEditors() - }) - const createClient = () => { - const mockClient = stub(DefaultCodeWhispererClient) - - mockClient.createCodeScan.resolves(mockCreateCodeScanResponse) - mockClient.createUploadUrl.resolves(mockCreateUploadUrlResponse) - mockClient.getCodeScan.resolves(mockGetCodeScanResponse) - mockClient.listCodeScanFindings.resolves(mockListCodeScanFindingsResponse) - return mockClient - } - - const openTestFile = async (filePath: string) => { - const doc = await vscode.workspace.openTextDocument(filePath) - return await vscode.window.showTextDocument(doc, { - selection: new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)), - }) - } - - it('Should render security scan result', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const commandSpy = sinon.spy(vscode.commands, 'executeCommand') - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - assert.ok(commandSpy.calledWith('workbench.action.problems.focus')) - assert.ok(securityScanRenderSpy.calledOnce) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - }) - - it('Should not focus problems panel for file scans', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const commandSpy = sinon.spy(vscode.commands, 'executeCommand') - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - - await model.CodeScansState.instance.setScansEnabled(true) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - assert.ok(commandSpy.neverCalledWith('workbench.action.problems.focus')) - assert.ok(securityScanRenderSpy.calledOnce) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'FILE', - passive: true, - }) - }) - - it('Should stop security scan for project scans when confirmed', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') - const testWindow = getTestWindow() - testWindow.onDidShowMessage((message) => { - if (message.message === stopScanMessage) { - message.selectItem(startSecurityScan.stopScanButton) - } - }) - model.codeScanState.setToRunning() - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - await startSecurityScan.confirmStopSecurityScan() - await scanPromise - assert.ok(securityScanRenderSpy.notCalled) - assert.ok(securityScanStoppedErrorSpy.calledOnce) - const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) - }) - - it('Should not stop security scan for project scans when not confirmed', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') - const testWindow = getTestWindow() - testWindow.onDidShowMessage((message) => { - if (message.message === stopScanMessage) { - message.selectItem(cancel) - } - }) - model.codeScanState.setToRunning() - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - await startSecurityScan.confirmStopSecurityScan() - await scanPromise - assert.ok(securityScanRenderSpy.calledOnce) - assert.ok(securityScanStoppedErrorSpy.notCalled) - const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) - }) - - it('Should stop security scan for file scans if setting is disabled', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') - await model.CodeScansState.instance.setScansEnabled(true) - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await model.CodeScansState.instance.setScansEnabled(false) - await scanPromise - assert.ok(securityScanRenderSpy.notCalled) - assert.ok(securityScanStoppedErrorSpy.calledOnce) - }) - - it('Should highlight files after scan is completed', async function () { - if (semver.lt(vscode.version, '1.78.0')) { - this.skip() - } - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const testWindow = getTestWindow() - testWindow.onDidShowMessage((message) => { - if (message.message.includes('Security scan completed')) { - message.selectItem(showScannedFilesMessage) - } - }) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanTotalIssues: 1, - codewhispererCodeScanIssuesWithFixes: 0, - codewhispererCodeScanScope: 'PROJECT', - passive: false, - } as CodewhispererSecurityScan) - }) - - it('Should cancel a scan if a newer one has started', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - await model.CodeScansState.instance.setScansEnabled(true) - - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await scanPromise - assertTelemetry('codewhisperer_securityScan', [ - { - result: 'Cancelled', - reasonDesc: 'Security scan stopped by user.', - reason: 'DefaultError', - } as unknown as CodewhispererSecurityScan, - { - result: 'Succeeded', - }, - ]) - }) - - it('Should not cancel a project scan if a file scan has started', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - await model.CodeScansState.instance.setScansEnabled(true) - - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await scanPromise - assertTelemetry('codewhisperer_securityScan', [ - { - result: 'Succeeded', - codewhispererCodeScanScope: 'FILE', - }, - { - result: 'Succeeded', - codewhispererCodeScanScope: 'PROJECT', - }, - ]) - }) - - it('Should handle failed scan job status', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - - const mockClient = createClient() - mockClient.getCodeScan.resolves({ - ...mockGetCodeScanResponse, - status: 'Failed', - }) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - mockClient, - extensionContext, - CodeAnalysisScope.PROJECT - ) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'PROJECT', - result: 'Failed', - reason: 'CodeScanJobFailedError', - reasonDesc: 'CodeScanJobFailedError: Security scan failed.', - passive: false, - } as unknown as CodewhispererSecurityScan) - }) - - it('Should show notification when throttled for project scans', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const mockClient = createClient() - mockClient.createCodeScan.throws({ - code: 'ThrottlingException', - time: new Date(), - name: 'error name', - message: 'Maximum project scan count reached for this month.', - } satisfies AWSError) - sinon.stub(errors, 'isAwsError').returns(true) - const testWindow = getTestWindow() - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - mockClient, - extensionContext, - CodeAnalysisScope.PROJECT - ) - assert.ok(testWindow.shownMessages.map((m) => m.message).includes(projectScansLimitReached)) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'PROJECT', - result: 'Failed', - reason: 'ThrottlingException', - reasonDesc: 'ThrottlingException: Maximum project scan count reached for this month.', - passive: false, - } as unknown as CodewhispererSecurityScan) - }) - - it('Should set monthly quota exceeded when throttled for file scans', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - await model.CodeScansState.instance.setScansEnabled(true) - const mockClient = createClient() - mockClient.createCodeScan.throws({ - code: 'ThrottlingException', - time: new Date(), - name: 'error name', - message: 'Maximum auto-scans count reached for this month.', - } satisfies AWSError) - sinon.stub(errors, 'isAwsError').returns(true) - assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), false) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - mockClient, - extensionContext, - CodeAnalysisScope.FILE - ) - assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), true) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'FILE', - result: 'Failed', - reason: 'ThrottlingException', - reasonDesc: 'ThrottlingException: Maximum auto-scans count reached for this month.', - passive: true, - } as unknown as CodewhispererSecurityScan) - }) -}) - describe('startSecurityScanPerformanceTest', function () { + let extensionContext: FakeExtensionContext + let mockSecurityPanelViewProvider: SecurityPanelViewProvider + let appCodePath: string + let editor: vscode.TextEditor beforeEach(async function () { extensionContext = await FakeExtensionContext.create() mockSecurityPanelViewProvider = new SecurityPanelViewProvider(extensionContext) From 3c1f90978a10d487b6be44db15d1f2f088dbc818 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 09:18:38 -0400 Subject: [PATCH 04/27] remove shared code to utils --- .../startSecurityScan.test.ts | 102 ++---------------- .../core/src/test/amazonqFeatureDev/utils.ts | 96 +++++++++++++++++ .../codewhisperer/startSecurityScan.test.ts | 101 ++--------------- 3 files changed, 109 insertions(+), 190 deletions(-) diff --git a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts index 7f668398992..a7df4790cd5 100644 --- a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts +++ b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts @@ -23,105 +23,17 @@ import { join } from 'path' import { assertTelemetry, closeAllEditors, getFetchStubWithResponse } from '../testUtil' import { stub } from '../../test/utilities/stubber' import { DefaultCodeWhispererClient } from '../../codewhisperer' -import { AWSError, HttpResponse } from 'aws-sdk' +import { AWSError } from 'aws-sdk' import { CodeAnalysisScope } from '../../codewhisperer' import { getTestWindow } from '../shared/vscode/window' import { SeverityLevel } from '../../test/shared/vscode/message' import { cancel, CodewhispererSecurityScan } from '../../shared' - -const mockCreateCodeScanResponse = { - $response: { - data: { - jobId: 'jobId', - status: 'Pending', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - jobId: 'jobId', - status: 'Pending', -} - -const mockCreateUploadUrlResponse = { - $response: { - data: { - uploadId: 'uploadId', - uploadUrl: 'uploadUrl', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - uploadId: 'uploadId', - uploadUrl: 'https://test.com', -} - -const mockGetCodeScanResponse = { - $response: { - data: { - status: 'Completed', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - status: 'Completed', -} - -const mockCodeScanFindings = JSON.stringify([ - { - filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', - startLine: 1, - endLine: 1, - title: 'title', - description: { - text: 'text', - markdown: 'markdown', - }, - detectorId: 'detectorId', - detectorName: 'detectorName', - findingId: 'findingId', - relatedVulnerabilities: [], - severity: 'High', - remediation: { - recommendation: { - text: 'text', - url: 'url', - }, - suggestedFixes: [], - }, - codeSnippet: [], - } satisfies model.RawCodeScanIssue, -]) - -const mockListCodeScanFindingsResponse = { - $response: { - data: { - codeScanFindings: mockCodeScanFindings, - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - codeScanFindings: mockCodeScanFindings, -} +import { + mockCreateCodeScanResponse, + mockCreateUploadUrlResponse, + mockGetCodeScanResponse, + mockListCodeScanFindingsResponse, +} from './utils' describe('startSecurityScan', function () { let extensionContext: FakeExtensionContext diff --git a/packages/core/src/test/amazonqFeatureDev/utils.ts b/packages/core/src/test/amazonqFeatureDev/utils.ts index 57e5fb72a9d..4bb9a219bce 100644 --- a/packages/core/src/test/amazonqFeatureDev/utils.ts +++ b/packages/core/src/test/amazonqFeatureDev/utils.ts @@ -5,6 +5,7 @@ import * as vscode from 'vscode' import * as sinon from 'sinon' +import * as model from '../../codewhisperer/models/model' import { MessagePublisher } from '../../amazonq/messages/messagePublisher' import { Messenger } from '../../amazonqFeatureDev/controllers/chat/messenger/messenger' import { AppToWebViewMessageDispatcher } from '../../amazonqFeatureDev/views/connector/connector' @@ -18,6 +19,7 @@ import { FeatureDevClient } from '../../amazonqFeatureDev/client/featureDev' import { VirtualMemoryFile } from '../../shared/virtualMemoryFile' import path from 'path' import { featureDevScheme } from '../../amazonqFeatureDev/constants' +import { HttpResponse } from 'aws-sdk' export function createMessenger(): Messenger { return new Messenger( @@ -116,3 +118,97 @@ export async function createController(): Promise { sessionStorage, } } + +export const mockCreateCodeScanResponse = { + $response: { + data: { + jobId: 'jobId', + status: 'Pending', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + jobId: 'jobId', + status: 'Pending', +} + +export const mockCreateUploadUrlResponse = { + $response: { + data: { + uploadId: 'uploadId', + uploadUrl: 'uploadUrl', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + uploadId: 'uploadId', + uploadUrl: 'https://test.com', +} + +export const mockGetCodeScanResponse = { + $response: { + data: { + status: 'Completed', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + status: 'Completed', +} + +export const mockCodeScanFindings = JSON.stringify([ + { + filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', + startLine: 1, + endLine: 1, + title: 'title', + description: { + text: 'text', + markdown: 'markdown', + }, + detectorId: 'detectorId', + detectorName: 'detectorName', + findingId: 'findingId', + relatedVulnerabilities: [], + severity: 'High', + remediation: { + recommendation: { + text: 'text', + url: 'url', + }, + suggestedFixes: [], + }, + codeSnippet: [], + } satisfies model.RawCodeScanIssue, +]) + +export const mockListCodeScanFindingsResponse = { + $response: { + data: { + codeScanFindings: mockCodeScanFindings, + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + codeScanFindings: mockCodeScanFindings, +} diff --git a/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts index c07cce545b3..0d64c9aa7ba 100644 --- a/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts @@ -21,105 +21,16 @@ import { toFile, } from '../../test/testUtil' import { stub } from '../../test/utilities/stubber' -import { HttpResponse } from 'aws-sdk' import { getTestWindow } from '../../test/shared/vscode/window' import { SeverityLevel } from '../../test/shared/vscode/message' import { CodeAnalysisScope } from '../../codewhisperer' import { performanceTest } from '../../shared/performance/performance' - -const mockCreateCodeScanResponse = { - $response: { - data: { - jobId: 'jobId', - status: 'Pending', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - jobId: 'jobId', - status: 'Pending', -} - -const mockCreateUploadUrlResponse = { - $response: { - data: { - uploadId: 'uploadId', - uploadUrl: 'uploadUrl', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - uploadId: 'uploadId', - uploadUrl: 'https://test.com', -} - -const mockGetCodeScanResponse = { - $response: { - data: { - status: 'Completed', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - status: 'Completed', -} - -const mockCodeScanFindings = JSON.stringify([ - { - filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', - startLine: 1, - endLine: 1, - title: 'title', - description: { - text: 'text', - markdown: 'markdown', - }, - detectorId: 'detectorId', - detectorName: 'detectorName', - findingId: 'findingId', - relatedVulnerabilities: [], - severity: 'High', - remediation: { - recommendation: { - text: 'text', - url: 'url', - }, - suggestedFixes: [], - }, - codeSnippet: [], - } satisfies model.RawCodeScanIssue, -]) - -const mockListCodeScanFindingsResponse = { - $response: { - data: { - codeScanFindings: mockCodeScanFindings, - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - codeScanFindings: mockCodeScanFindings, -} +import { + mockCreateCodeScanResponse, + mockCreateUploadUrlResponse, + mockGetCodeScanResponse, + mockListCodeScanFindingsResponse, +} from '../../test/amazonqFeatureDev/utils' describe('startSecurityScanPerformanceTest', function () { let extensionContext: FakeExtensionContext From f5d6c60e98731add4ba736bb7c17fb4eb9773e04 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 09:24:17 -0400 Subject: [PATCH 05/27] move out more shared code --- .../startSecurityScan.test.ts | 44 ++--- .../core/src/test/amazonqFeatureDev/utils.ts | 153 ++++++++++-------- .../codewhisperer/startSecurityScan.test.ts | 20 +-- 3 files changed, 98 insertions(+), 119 deletions(-) diff --git a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts index a7df4790cd5..42735ebd464 100644 --- a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts +++ b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts @@ -21,19 +21,12 @@ import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilitie import { FakeExtensionContext } from '../fakeExtensionContext' import { join } from 'path' import { assertTelemetry, closeAllEditors, getFetchStubWithResponse } from '../testUtil' -import { stub } from '../../test/utilities/stubber' -import { DefaultCodeWhispererClient } from '../../codewhisperer' import { AWSError } from 'aws-sdk' import { CodeAnalysisScope } from '../../codewhisperer' import { getTestWindow } from '../shared/vscode/window' import { SeverityLevel } from '../../test/shared/vscode/message' import { cancel, CodewhispererSecurityScan } from '../../shared' -import { - mockCreateCodeScanResponse, - mockCreateUploadUrlResponse, - mockGetCodeScanResponse, - mockListCodeScanFindingsResponse, -} from './utils' +import { createMockClient, mockGetCodeScanResponse } from './utils' describe('startSecurityScan', function () { let extensionContext: FakeExtensionContext @@ -57,15 +50,6 @@ describe('startSecurityScan', function () { after(async function () { await closeAllEditors() }) - const createClient = () => { - const mockClient = stub(DefaultCodeWhispererClient) - - mockClient.createCodeScan.resolves(mockCreateCodeScanResponse) - mockClient.createUploadUrl.resolves(mockCreateUploadUrlResponse) - mockClient.getCodeScan.resolves(mockGetCodeScanResponse) - mockClient.listCodeScanFindings.resolves(mockListCodeScanFindingsResponse) - return mockClient - } const openTestFile = async (filePath: string) => { const doc = await vscode.workspace.openTextDocument(filePath) @@ -82,7 +66,7 @@ describe('startSecurityScan', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -101,7 +85,7 @@ describe('startSecurityScan', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -129,7 +113,7 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -155,7 +139,7 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -175,7 +159,7 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -199,7 +183,7 @@ describe('startSecurityScan', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -218,14 +202,14 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.FILE ) await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -249,14 +233,14 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.PROJECT ) await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -276,7 +260,7 @@ describe('startSecurityScan', function () { it('Should handle failed scan job status', async function () { getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const mockClient = createClient() + const mockClient = createMockClient() mockClient.getCodeScan.resolves({ ...mockGetCodeScanResponse, status: 'Failed', @@ -299,7 +283,7 @@ describe('startSecurityScan', function () { it('Should show notification when throttled for project scans', async function () { getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const mockClient = createClient() + const mockClient = createMockClient() mockClient.createCodeScan.throws({ code: 'ThrottlingException', time: new Date(), @@ -328,7 +312,7 @@ describe('startSecurityScan', function () { it('Should set monthly quota exceeded when throttled for file scans', async function () { getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) await model.CodeScansState.instance.setScansEnabled(true) - const mockClient = createClient() + const mockClient = createMockClient() mockClient.createCodeScan.throws({ code: 'ThrottlingException', time: new Date(), diff --git a/packages/core/src/test/amazonqFeatureDev/utils.ts b/packages/core/src/test/amazonqFeatureDev/utils.ts index 4bb9a219bce..87ec6fc09c6 100644 --- a/packages/core/src/test/amazonqFeatureDev/utils.ts +++ b/packages/core/src/test/amazonqFeatureDev/utils.ts @@ -20,6 +20,8 @@ import { VirtualMemoryFile } from '../../shared/virtualMemoryFile' import path from 'path' import { featureDevScheme } from '../../amazonqFeatureDev/constants' import { HttpResponse } from 'aws-sdk' +import { DefaultCodeWhispererClient } from '../../codewhisperer' +import { stub } from '../utilities/stubber' export function createMessenger(): Messenger { return new Messenger( @@ -119,42 +121,6 @@ export async function createController(): Promise { } } -export const mockCreateCodeScanResponse = { - $response: { - data: { - jobId: 'jobId', - status: 'Pending', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - jobId: 'jobId', - status: 'Pending', -} - -export const mockCreateUploadUrlResponse = { - $response: { - data: { - uploadId: 'uploadId', - uploadUrl: 'uploadUrl', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - uploadId: 'uploadId', - uploadUrl: 'https://test.com', -} - export const mockGetCodeScanResponse = { $response: { data: { @@ -171,44 +137,89 @@ export const mockGetCodeScanResponse = { status: 'Completed', } -export const mockCodeScanFindings = JSON.stringify([ - { - filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', - startLine: 1, - endLine: 1, - title: 'title', - description: { - text: 'text', - markdown: 'markdown', +export function createMockClient() { + const mockClient = stub(DefaultCodeWhispererClient) + + const mockCreateCodeScanResponse = { + $response: { + data: { + jobId: 'jobId', + status: 'Pending', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), }, - detectorId: 'detectorId', - detectorName: 'detectorName', - findingId: 'findingId', - relatedVulnerabilities: [], - severity: 'High', - remediation: { - recommendation: { - text: 'text', - url: 'url', + jobId: 'jobId', + status: 'Pending', + } + const mockCreateUploadUrlResponse = { + $response: { + data: { + uploadId: 'uploadId', + uploadUrl: 'uploadUrl', }, - suggestedFixes: [], + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), }, - codeSnippet: [], - } satisfies model.RawCodeScanIssue, -]) + uploadId: 'uploadId', + uploadUrl: 'https://test.com', + } -export const mockListCodeScanFindingsResponse = { - $response: { - data: { - codeScanFindings: mockCodeScanFindings, + const mockCodeScanFindings = JSON.stringify([ + { + filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', + startLine: 1, + endLine: 1, + title: 'title', + description: { + text: 'text', + markdown: 'markdown', + }, + detectorId: 'detectorId', + detectorName: 'detectorName', + findingId: 'findingId', + relatedVulnerabilities: [], + severity: 'High', + remediation: { + recommendation: { + text: 'text', + url: 'url', + }, + suggestedFixes: [], + }, + codeSnippet: [], + } satisfies model.RawCodeScanIssue, + ]) + + const mockListCodeScanFindingsResponse = { + $response: { + data: { + codeScanFindings: mockCodeScanFindings, + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - codeScanFindings: mockCodeScanFindings, + codeScanFindings: mockCodeScanFindings, + } + + mockClient.createCodeScan.resolves(mockCreateCodeScanResponse) + mockClient.createUploadUrl.resolves(mockCreateUploadUrlResponse) + mockClient.getCodeScan.resolves(mockGetCodeScanResponse) + mockClient.listCodeScanFindings.resolves(mockListCodeScanFindingsResponse) + return mockClient } diff --git a/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts index 0d64c9aa7ba..ff5da892e90 100644 --- a/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts @@ -9,7 +9,6 @@ import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsPro import * as model from '../../codewhisperer/models/model' import * as timeoutUtils from '../../shared/utilities/timeoutUtils' import assert from 'assert' -import { DefaultCodeWhispererClient } from '../../codewhisperer' import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' import { FakeExtensionContext } from '../../test/fakeExtensionContext' import { join } from 'path' @@ -20,17 +19,11 @@ import { getFetchStubWithResponse, toFile, } from '../../test/testUtil' -import { stub } from '../../test/utilities/stubber' import { getTestWindow } from '../../test/shared/vscode/window' import { SeverityLevel } from '../../test/shared/vscode/message' import { CodeAnalysisScope } from '../../codewhisperer' import { performanceTest } from '../../shared/performance/performance' -import { - mockCreateCodeScanResponse, - mockCreateUploadUrlResponse, - mockGetCodeScanResponse, - mockListCodeScanFindingsResponse, -} from '../../test/amazonqFeatureDev/utils' +import { createMockClient } from '../../test/amazonqFeatureDev/utils' describe('startSecurityScanPerformanceTest', function () { let extensionContext: FakeExtensionContext @@ -57,15 +50,6 @@ describe('startSecurityScanPerformanceTest', function () { await closeAllEditors() }) - const createClient = () => { - const mockClient = stub(DefaultCodeWhispererClient) - mockClient.createCodeScan.resolves(mockCreateCodeScanResponse) - mockClient.createUploadUrl.resolves(mockCreateUploadUrlResponse) - mockClient.getCodeScan.resolves(mockGetCodeScanResponse) - mockClient.listCodeScanFindings.resolves(mockListCodeScanFindingsResponse) - return mockClient - } - const openTestFile = async (filePath: string) => { const doc = await vscode.workspace.openTextDocument(filePath) return await vscode.window.showTextDocument(doc, { @@ -86,7 +70,7 @@ describe('startSecurityScanPerformanceTest', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createClient(), + createMockClient(), extensionContext, CodeAnalysisScope.FILE ) From caf2be83bebe3cf17841a3366b2ebf221cbb2730 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 15:25:08 -0400 Subject: [PATCH 06/27] move some files around --- .../codewhisperer/startSecurityScan.test.ts | 342 ++++++++++++++++++ .../startSecurityScan.test.ts | 0 2 files changed, 342 insertions(+) create mode 100644 packages/core/src/test/codewhisperer/startSecurityScan.test.ts rename packages/core/src/testInteg/{codewhisperer => }/startSecurityScan.test.ts (100%) diff --git a/packages/core/src/test/codewhisperer/startSecurityScan.test.ts b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts new file mode 100644 index 00000000000..873b6ad4307 --- /dev/null +++ b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts @@ -0,0 +1,342 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import * as vscode from 'vscode' +import * as sinon from 'sinon' +import * as assert from 'assert' +import * as semver from 'semver' +import * as model from '../../codewhisperer/models/model' +import * as timeoutUtils from '../../shared/utilities/timeoutUtils' +import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' +import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' +import * as errors from '../../shared/errors' +import { + SecurityPanelViewProvider, + stopScanMessage, + showScannedFilesMessage, + projectScansLimitReached, +} from '../../codewhisperer' +import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' +import { FakeExtensionContext } from '../fakeExtensionContext' +import { join } from 'path' +import { assertTelemetry, closeAllEditors, getFetchStubWithResponse } from '../testUtil' +import { AWSError } from 'aws-sdk' +import { CodeAnalysisScope } from '../../codewhisperer' +import { getTestWindow } from '../shared/vscode/window' +import { SeverityLevel } from '../../test/shared/vscode/message' +import { cancel, CodewhispererSecurityScan } from '../../shared' +import { createMockClient, mockGetCodeScanResponse } from '../amazonqFeatureDev/utils' + +describe('startSecurityScan', function () { + let extensionContext: FakeExtensionContext + let mockSecurityPanelViewProvider: SecurityPanelViewProvider + let appRoot: string + let appCodePath: string + let editor: vscode.TextEditor + const workspaceFolder = getTestWorkspaceFolder() + beforeEach(async function () { + extensionContext = await FakeExtensionContext.create() + mockSecurityPanelViewProvider = new SecurityPanelViewProvider(extensionContext) + appRoot = join(workspaceFolder, 'python3.7-plain-sam-app') + appCodePath = join(appRoot, 'hello_world', 'app.py') + editor = await openTestFile(appCodePath) + await model.CodeScansState.instance.setScansEnabled(false) + sinon.stub(timeoutUtils, 'sleep') + }) + afterEach(function () { + sinon.restore() + }) + after(async function () { + await closeAllEditors() + }) + + const openTestFile = async (filePath: string) => { + const doc = await vscode.workspace.openTextDocument(filePath) + return await vscode.window.showTextDocument(doc, { + selection: new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)), + }) + } + + it('Should render security scan result', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const commandSpy = sinon.spy(vscode.commands, 'executeCommand') + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + assert.ok(commandSpy.calledWith('workbench.action.problems.focus')) + assert.ok(securityScanRenderSpy.calledOnce) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + }) + + it('Should not focus problems panel for file scans', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const commandSpy = sinon.spy(vscode.commands, 'executeCommand') + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + + await model.CodeScansState.instance.setScansEnabled(true) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + assert.ok(commandSpy.neverCalledWith('workbench.action.problems.focus')) + assert.ok(securityScanRenderSpy.calledOnce) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'FILE', + passive: true, + }) + }) + + it('Should stop security scan for project scans when confirmed', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') + const testWindow = getTestWindow() + testWindow.onDidShowMessage((message) => { + if (message.message === stopScanMessage) { + message.selectItem(startSecurityScan.stopScanButton) + } + }) + model.codeScanState.setToRunning() + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + await startSecurityScan.confirmStopSecurityScan() + await scanPromise + assert.ok(securityScanRenderSpy.notCalled) + assert.ok(securityScanStoppedErrorSpy.calledOnce) + const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) + }) + + it('Should not stop security scan for project scans when not confirmed', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') + const testWindow = getTestWindow() + testWindow.onDidShowMessage((message) => { + if (message.message === stopScanMessage) { + message.selectItem(cancel) + } + }) + model.codeScanState.setToRunning() + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + await startSecurityScan.confirmStopSecurityScan() + await scanPromise + assert.ok(securityScanRenderSpy.calledOnce) + assert.ok(securityScanStoppedErrorSpy.notCalled) + const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) + }) + + it('Should stop security scan for file scans if setting is disabled', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') + await model.CodeScansState.instance.setScansEnabled(true) + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await model.CodeScansState.instance.setScansEnabled(false) + await scanPromise + assert.ok(securityScanRenderSpy.notCalled) + assert.ok(securityScanStoppedErrorSpy.calledOnce) + }) + + it('Should highlight files after scan is completed', async function () { + if (semver.lt(vscode.version, '1.78.0')) { + this.skip() + } + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const testWindow = getTestWindow() + testWindow.onDidShowMessage((message) => { + if (message.message.includes('Security scan completed')) { + message.selectItem(showScannedFilesMessage) + } + }) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanTotalIssues: 1, + codewhispererCodeScanIssuesWithFixes: 0, + codewhispererCodeScanScope: 'PROJECT', + passive: false, + } as CodewhispererSecurityScan) + }) + + it('Should cancel a scan if a newer one has started', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + await model.CodeScansState.instance.setScansEnabled(true) + + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await scanPromise + assertTelemetry('codewhisperer_securityScan', [ + { + result: 'Cancelled', + reasonDesc: 'Security scan stopped by user.', + reason: 'DefaultError', + } as unknown as CodewhispererSecurityScan, + { + result: 'Succeeded', + }, + ]) + }) + + it('Should not cancel a project scan if a file scan has started', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + await model.CodeScansState.instance.setScansEnabled(true) + + const scanPromise = startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.PROJECT + ) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createMockClient(), + extensionContext, + CodeAnalysisScope.FILE + ) + await scanPromise + assertTelemetry('codewhisperer_securityScan', [ + { + result: 'Succeeded', + codewhispererCodeScanScope: 'FILE', + }, + { + result: 'Succeeded', + codewhispererCodeScanScope: 'PROJECT', + }, + ]) + }) + + it('Should handle failed scan job status', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + + const mockClient = createMockClient() + mockClient.getCodeScan.resolves({ + ...mockGetCodeScanResponse, + status: 'Failed', + }) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + mockClient, + extensionContext, + CodeAnalysisScope.PROJECT + ) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'PROJECT', + result: 'Failed', + reason: 'CodeScanJobFailedError', + reasonDesc: 'CodeScanJobFailedError: Security scan failed.', + passive: false, + } as unknown as CodewhispererSecurityScan) + }) + + it('Should show notification when throttled for project scans', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const mockClient = createMockClient() + mockClient.createCodeScan.throws({ + code: 'ThrottlingException', + time: new Date(), + name: 'error name', + message: 'Maximum project scan count reached for this month.', + } satisfies AWSError) + sinon.stub(errors, 'isAwsError').returns(true) + const testWindow = getTestWindow() + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + mockClient, + extensionContext, + CodeAnalysisScope.PROJECT + ) + assert.ok(testWindow.shownMessages.map((m) => m.message).includes(projectScansLimitReached)) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'PROJECT', + result: 'Failed', + reason: 'ThrottlingException', + reasonDesc: 'ThrottlingException: Maximum project scan count reached for this month.', + passive: false, + } as unknown as CodewhispererSecurityScan) + }) + + it('Should set monthly quota exceeded when throttled for file scans', async function () { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + await model.CodeScansState.instance.setScansEnabled(true) + const mockClient = createMockClient() + mockClient.createCodeScan.throws({ + code: 'ThrottlingException', + time: new Date(), + name: 'error name', + message: 'Maximum auto-scans count reached for this month.', + } satisfies AWSError) + sinon.stub(errors, 'isAwsError').returns(true) + assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), false) + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + mockClient, + extensionContext, + CodeAnalysisScope.FILE + ) + assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), true) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'FILE', + result: 'Failed', + reason: 'ThrottlingException', + reasonDesc: 'ThrottlingException: Maximum auto-scans count reached for this month.', + passive: true, + } as unknown as CodewhispererSecurityScan) + }) +}) diff --git a/packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts b/packages/core/src/testInteg/startSecurityScan.test.ts similarity index 100% rename from packages/core/src/testInteg/codewhisperer/startSecurityScan.test.ts rename to packages/core/src/testInteg/startSecurityScan.test.ts From 658ed5ae892b8d1abf471c8d878ce3d17b1089a6 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 16:31:56 -0400 Subject: [PATCH 07/27] update imports --- .../src/testInteg/startSecurityScan.test.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/core/src/testInteg/startSecurityScan.test.ts b/packages/core/src/testInteg/startSecurityScan.test.ts index ff5da892e90..d3a17a1650a 100644 --- a/packages/core/src/testInteg/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/startSecurityScan.test.ts @@ -4,13 +4,13 @@ */ import * as vscode from 'vscode' import * as sinon from 'sinon' -import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' -import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' -import * as model from '../../codewhisperer/models/model' -import * as timeoutUtils from '../../shared/utilities/timeoutUtils' +import * as startSecurityScan from '../codewhisperer/commands/startSecurityScan' +import * as diagnosticsProvider from '../codewhisperer/service/diagnosticsProvider' +import * as model from '../codewhisperer/models/model' +import * as timeoutUtils from '../shared/utilities/timeoutUtils' import assert from 'assert' -import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' -import { FakeExtensionContext } from '../../test/fakeExtensionContext' +import { SecurityPanelViewProvider } from '../codewhisperer/views/securityPanelViewProvider' +import { FakeExtensionContext } from '../test/fakeExtensionContext' import { join } from 'path' import { assertTelemetry, @@ -18,12 +18,12 @@ import { createTestWorkspaceFolder, getFetchStubWithResponse, toFile, -} from '../../test/testUtil' -import { getTestWindow } from '../../test/shared/vscode/window' -import { SeverityLevel } from '../../test/shared/vscode/message' -import { CodeAnalysisScope } from '../../codewhisperer' -import { performanceTest } from '../../shared/performance/performance' -import { createMockClient } from '../../test/amazonqFeatureDev/utils' +} from '../test/testUtil' +import { getTestWindow } from '../test/shared/vscode/window' +import { SeverityLevel } from '../test/shared/vscode/message' +import { CodeAnalysisScope } from '../codewhisperer' +import { performanceTest } from '../shared/performance/performance' +import { createMockClient } from '../test/amazonqFeatureDev/utils' describe('startSecurityScanPerformanceTest', function () { let extensionContext: FakeExtensionContext From 366b0386fcde6d40f5f95e962ba60c4baecd44f7 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 17:13:21 -0400 Subject: [PATCH 08/27] fix test changes --- .../startSecurityScan.test.ts | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts index 42735ebd464..718769eaa22 100644 --- a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts +++ b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts @@ -2,38 +2,41 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ + +import assert from 'assert' import * as vscode from 'vscode' import * as sinon from 'sinon' -import * as assert from 'assert' import * as semver from 'semver' -import * as model from '../../codewhisperer/models/model' -import * as timeoutUtils from '../../shared/utilities/timeoutUtils' -import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' -import * as errors from '../../shared/errors' -import { - SecurityPanelViewProvider, - stopScanMessage, - showScannedFilesMessage, - projectScansLimitReached, -} from '../../codewhisperer' -import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' +import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' import { FakeExtensionContext } from '../fakeExtensionContext' +import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' +import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' import { join } from 'path' import { assertTelemetry, closeAllEditors, getFetchStubWithResponse } from '../testUtil' import { AWSError } from 'aws-sdk' -import { CodeAnalysisScope } from '../../codewhisperer' import { getTestWindow } from '../shared/vscode/window' -import { SeverityLevel } from '../../test/shared/vscode/message' -import { cancel, CodewhispererSecurityScan } from '../../shared' +import { SeverityLevel } from '../shared/vscode/message' +import { cancel } from '../../shared/localizedText' +import { + showScannedFilesMessage, + stopScanMessage, + CodeAnalysisScope, + projectScansLimitReached, +} from '../../codewhisperer/models/constants' +import * as model from '../../codewhisperer/models/model' +import { CodewhispererSecurityScan } from '../../shared/telemetry/telemetry.gen' +import * as errors from '../../shared/errors' +import * as timeoutUtils from '../../shared/utilities/timeoutUtils' import { createMockClient, mockGetCodeScanResponse } from './utils' +let extensionContext: FakeExtensionContext +let mockSecurityPanelViewProvider: SecurityPanelViewProvider +let appRoot: string +let appCodePath: string +let editor: vscode.TextEditor + describe('startSecurityScan', function () { - let extensionContext: FakeExtensionContext - let mockSecurityPanelViewProvider: SecurityPanelViewProvider - let appRoot: string - let appCodePath: string - let editor: vscode.TextEditor const workspaceFolder = getTestWorkspaceFolder() beforeEach(async function () { extensionContext = await FakeExtensionContext.create() From 1786ff497d56856e33c6d8119230c8e8679dc9b7 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 17:30:46 -0400 Subject: [PATCH 09/27] delete duplicate test file --- .../startSecurityScan.test.ts | 345 ------------------ .../codewhisperer/startSecurityScan.test.ts | 45 +-- 2 files changed, 24 insertions(+), 366 deletions(-) delete mode 100644 packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts diff --git a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts b/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts deleted file mode 100644 index 718769eaa22..00000000000 --- a/packages/core/src/test/amazonqFeatureDev/startSecurityScan.test.ts +++ /dev/null @@ -1,345 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import assert from 'assert' -import * as vscode from 'vscode' -import * as sinon from 'sinon' -import * as semver from 'semver' -import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' -import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' -import { FakeExtensionContext } from '../fakeExtensionContext' -import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' -import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' -import { join } from 'path' -import { assertTelemetry, closeAllEditors, getFetchStubWithResponse } from '../testUtil' -import { AWSError } from 'aws-sdk' -import { getTestWindow } from '../shared/vscode/window' -import { SeverityLevel } from '../shared/vscode/message' -import { cancel } from '../../shared/localizedText' -import { - showScannedFilesMessage, - stopScanMessage, - CodeAnalysisScope, - projectScansLimitReached, -} from '../../codewhisperer/models/constants' -import * as model from '../../codewhisperer/models/model' -import { CodewhispererSecurityScan } from '../../shared/telemetry/telemetry.gen' -import * as errors from '../../shared/errors' -import * as timeoutUtils from '../../shared/utilities/timeoutUtils' -import { createMockClient, mockGetCodeScanResponse } from './utils' - -let extensionContext: FakeExtensionContext -let mockSecurityPanelViewProvider: SecurityPanelViewProvider -let appRoot: string -let appCodePath: string -let editor: vscode.TextEditor - -describe('startSecurityScan', function () { - const workspaceFolder = getTestWorkspaceFolder() - beforeEach(async function () { - extensionContext = await FakeExtensionContext.create() - mockSecurityPanelViewProvider = new SecurityPanelViewProvider(extensionContext) - appRoot = join(workspaceFolder, 'python3.7-plain-sam-app') - appCodePath = join(appRoot, 'hello_world', 'app.py') - editor = await openTestFile(appCodePath) - await model.CodeScansState.instance.setScansEnabled(false) - sinon.stub(timeoutUtils, 'sleep') - }) - afterEach(function () { - sinon.restore() - }) - after(async function () { - await closeAllEditors() - }) - - const openTestFile = async (filePath: string) => { - const doc = await vscode.workspace.openTextDocument(filePath) - return await vscode.window.showTextDocument(doc, { - selection: new vscode.Range(new vscode.Position(0, 0), new vscode.Position(0, 1)), - }) - } - - it('Should render security scan result', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const commandSpy = sinon.spy(vscode.commands, 'executeCommand') - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - assert.ok(commandSpy.calledWith('workbench.action.problems.focus')) - assert.ok(securityScanRenderSpy.calledOnce) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - }) - - it('Should not focus problems panel for file scans', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const commandSpy = sinon.spy(vscode.commands, 'executeCommand') - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - - await model.CodeScansState.instance.setScansEnabled(true) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - assert.ok(commandSpy.neverCalledWith('workbench.action.problems.focus')) - assert.ok(securityScanRenderSpy.calledOnce) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'FILE', - passive: true, - }) - }) - - it('Should stop security scan for project scans when confirmed', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') - const testWindow = getTestWindow() - testWindow.onDidShowMessage((message) => { - if (message.message === stopScanMessage) { - message.selectItem(startSecurityScan.stopScanButton) - } - }) - model.codeScanState.setToRunning() - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - await startSecurityScan.confirmStopSecurityScan() - await scanPromise - assert.ok(securityScanRenderSpy.notCalled) - assert.ok(securityScanStoppedErrorSpy.calledOnce) - const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) - }) - - it('Should not stop security scan for project scans when not confirmed', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') - const testWindow = getTestWindow() - testWindow.onDidShowMessage((message) => { - if (message.message === stopScanMessage) { - message.selectItem(cancel) - } - }) - model.codeScanState.setToRunning() - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - await startSecurityScan.confirmStopSecurityScan() - await scanPromise - assert.ok(securityScanRenderSpy.calledOnce) - assert.ok(securityScanStoppedErrorSpy.notCalled) - const warnings = testWindow.shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.ok(warnings.map((m) => m.message).includes(stopScanMessage)) - }) - - it('Should stop security scan for file scans if setting is disabled', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - const securityScanStoppedErrorSpy = sinon.spy(model, 'CodeScanStoppedError') - await model.CodeScansState.instance.setScansEnabled(true) - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await model.CodeScansState.instance.setScansEnabled(false) - await scanPromise - assert.ok(securityScanRenderSpy.notCalled) - assert.ok(securityScanStoppedErrorSpy.calledOnce) - }) - - it('Should highlight files after scan is completed', async function () { - if (semver.lt(vscode.version, '1.78.0')) { - this.skip() - } - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const testWindow = getTestWindow() - testWindow.onDidShowMessage((message) => { - if (message.message.includes('Security scan completed')) { - message.selectItem(showScannedFilesMessage) - } - }) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanTotalIssues: 1, - codewhispererCodeScanIssuesWithFixes: 0, - codewhispererCodeScanScope: 'PROJECT', - passive: false, - } as CodewhispererSecurityScan) - }) - - it('Should cancel a scan if a newer one has started', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - await model.CodeScansState.instance.setScansEnabled(true) - - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await scanPromise - assertTelemetry('codewhisperer_securityScan', [ - { - result: 'Cancelled', - reasonDesc: 'Security scan stopped by user.', - reason: 'DefaultError', - } as unknown as CodewhispererSecurityScan, - { - result: 'Succeeded', - }, - ]) - }) - - it('Should not cancel a project scan if a file scan has started', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - await model.CodeScansState.instance.setScansEnabled(true) - - const scanPromise = startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.PROJECT - ) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createMockClient(), - extensionContext, - CodeAnalysisScope.FILE - ) - await scanPromise - assertTelemetry('codewhisperer_securityScan', [ - { - result: 'Succeeded', - codewhispererCodeScanScope: 'FILE', - }, - { - result: 'Succeeded', - codewhispererCodeScanScope: 'PROJECT', - }, - ]) - }) - - it('Should handle failed scan job status', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - - const mockClient = createMockClient() - mockClient.getCodeScan.resolves({ - ...mockGetCodeScanResponse, - status: 'Failed', - }) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - mockClient, - extensionContext, - CodeAnalysisScope.PROJECT - ) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'PROJECT', - result: 'Failed', - reason: 'CodeScanJobFailedError', - reasonDesc: 'CodeScanJobFailedError: Security scan failed.', - passive: false, - } as unknown as CodewhispererSecurityScan) - }) - - it('Should show notification when throttled for project scans', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const mockClient = createMockClient() - mockClient.createCodeScan.throws({ - code: 'ThrottlingException', - time: new Date(), - name: 'error name', - message: 'Maximum project scan count reached for this month.', - } satisfies AWSError) - sinon.stub(errors, 'isAwsError').returns(true) - const testWindow = getTestWindow() - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - mockClient, - extensionContext, - CodeAnalysisScope.PROJECT - ) - assert.ok(testWindow.shownMessages.map((m) => m.message).includes(projectScansLimitReached)) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'PROJECT', - result: 'Failed', - reason: 'ThrottlingException', - reasonDesc: 'ThrottlingException: Maximum project scan count reached for this month.', - passive: false, - } as unknown as CodewhispererSecurityScan) - }) - - it('Should set monthly quota exceeded when throttled for file scans', async function () { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - await model.CodeScansState.instance.setScansEnabled(true) - const mockClient = createMockClient() - mockClient.createCodeScan.throws({ - code: 'ThrottlingException', - time: new Date(), - name: 'error name', - message: 'Maximum auto-scans count reached for this month.', - } satisfies AWSError) - sinon.stub(errors, 'isAwsError').returns(true) - assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), false) - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - mockClient, - extensionContext, - CodeAnalysisScope.FILE - ) - assert.equal(model.CodeScansState.instance.isMonthlyQuotaExceeded(), true) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'FILE', - result: 'Failed', - reason: 'ThrottlingException', - reasonDesc: 'ThrottlingException: Maximum auto-scans count reached for this month.', - passive: true, - } as unknown as CodewhispererSecurityScan) - }) -}) diff --git a/packages/core/src/test/codewhisperer/startSecurityScan.test.ts b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts index 873b6ad4307..718769eaa22 100644 --- a/packages/core/src/test/codewhisperer/startSecurityScan.test.ts +++ b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts @@ -2,38 +2,41 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ + +import assert from 'assert' import * as vscode from 'vscode' import * as sinon from 'sinon' -import * as assert from 'assert' import * as semver from 'semver' -import * as model from '../../codewhisperer/models/model' -import * as timeoutUtils from '../../shared/utilities/timeoutUtils' -import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' -import * as errors from '../../shared/errors' -import { - SecurityPanelViewProvider, - stopScanMessage, - showScannedFilesMessage, - projectScansLimitReached, -} from '../../codewhisperer' -import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' +import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' import { FakeExtensionContext } from '../fakeExtensionContext' +import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' +import { getTestWorkspaceFolder } from '../../testInteg/integrationTestsUtilities' import { join } from 'path' import { assertTelemetry, closeAllEditors, getFetchStubWithResponse } from '../testUtil' import { AWSError } from 'aws-sdk' -import { CodeAnalysisScope } from '../../codewhisperer' import { getTestWindow } from '../shared/vscode/window' -import { SeverityLevel } from '../../test/shared/vscode/message' -import { cancel, CodewhispererSecurityScan } from '../../shared' -import { createMockClient, mockGetCodeScanResponse } from '../amazonqFeatureDev/utils' +import { SeverityLevel } from '../shared/vscode/message' +import { cancel } from '../../shared/localizedText' +import { + showScannedFilesMessage, + stopScanMessage, + CodeAnalysisScope, + projectScansLimitReached, +} from '../../codewhisperer/models/constants' +import * as model from '../../codewhisperer/models/model' +import { CodewhispererSecurityScan } from '../../shared/telemetry/telemetry.gen' +import * as errors from '../../shared/errors' +import * as timeoutUtils from '../../shared/utilities/timeoutUtils' +import { createMockClient, mockGetCodeScanResponse } from './utils' + +let extensionContext: FakeExtensionContext +let mockSecurityPanelViewProvider: SecurityPanelViewProvider +let appRoot: string +let appCodePath: string +let editor: vscode.TextEditor describe('startSecurityScan', function () { - let extensionContext: FakeExtensionContext - let mockSecurityPanelViewProvider: SecurityPanelViewProvider - let appRoot: string - let appCodePath: string - let editor: vscode.TextEditor const workspaceFolder = getTestWorkspaceFolder() beforeEach(async function () { extensionContext = await FakeExtensionContext.create() From 3b987b7aeae03009b389a83400519ef63a05ce2d Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 17:36:09 -0400 Subject: [PATCH 10/27] fix tests again --- .../core/src/test/amazonqFeatureDev/utils.ts | 107 ------------------ .../codewhisperer/startSecurityScan.test.ts | 2 +- .../core/src/test/codewhisperer/testUtil.ts | 107 +++++++++++++++++- 3 files changed, 107 insertions(+), 109 deletions(-) diff --git a/packages/core/src/test/amazonqFeatureDev/utils.ts b/packages/core/src/test/amazonqFeatureDev/utils.ts index 87ec6fc09c6..57e5fb72a9d 100644 --- a/packages/core/src/test/amazonqFeatureDev/utils.ts +++ b/packages/core/src/test/amazonqFeatureDev/utils.ts @@ -5,7 +5,6 @@ import * as vscode from 'vscode' import * as sinon from 'sinon' -import * as model from '../../codewhisperer/models/model' import { MessagePublisher } from '../../amazonq/messages/messagePublisher' import { Messenger } from '../../amazonqFeatureDev/controllers/chat/messenger/messenger' import { AppToWebViewMessageDispatcher } from '../../amazonqFeatureDev/views/connector/connector' @@ -19,9 +18,6 @@ import { FeatureDevClient } from '../../amazonqFeatureDev/client/featureDev' import { VirtualMemoryFile } from '../../shared/virtualMemoryFile' import path from 'path' import { featureDevScheme } from '../../amazonqFeatureDev/constants' -import { HttpResponse } from 'aws-sdk' -import { DefaultCodeWhispererClient } from '../../codewhisperer' -import { stub } from '../utilities/stubber' export function createMessenger(): Messenger { return new Messenger( @@ -120,106 +116,3 @@ export async function createController(): Promise { sessionStorage, } } - -export const mockGetCodeScanResponse = { - $response: { - data: { - status: 'Completed', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - status: 'Completed', -} - -export function createMockClient() { - const mockClient = stub(DefaultCodeWhispererClient) - - const mockCreateCodeScanResponse = { - $response: { - data: { - jobId: 'jobId', - status: 'Pending', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - jobId: 'jobId', - status: 'Pending', - } - const mockCreateUploadUrlResponse = { - $response: { - data: { - uploadId: 'uploadId', - uploadUrl: 'uploadUrl', - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - uploadId: 'uploadId', - uploadUrl: 'https://test.com', - } - - const mockCodeScanFindings = JSON.stringify([ - { - filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', - startLine: 1, - endLine: 1, - title: 'title', - description: { - text: 'text', - markdown: 'markdown', - }, - detectorId: 'detectorId', - detectorName: 'detectorName', - findingId: 'findingId', - relatedVulnerabilities: [], - severity: 'High', - remediation: { - recommendation: { - text: 'text', - url: 'url', - }, - suggestedFixes: [], - }, - codeSnippet: [], - } satisfies model.RawCodeScanIssue, - ]) - - const mockListCodeScanFindingsResponse = { - $response: { - data: { - codeScanFindings: mockCodeScanFindings, - }, - requestId: 'requestId', - hasNextPage: () => false, - error: undefined, - nextPage: () => undefined, - redirectCount: 0, - retryCount: 0, - httpResponse: new HttpResponse(), - }, - codeScanFindings: mockCodeScanFindings, - } - - mockClient.createCodeScan.resolves(mockCreateCodeScanResponse) - mockClient.createUploadUrl.resolves(mockCreateUploadUrlResponse) - mockClient.getCodeScan.resolves(mockGetCodeScanResponse) - mockClient.listCodeScanFindings.resolves(mockListCodeScanFindingsResponse) - return mockClient -} diff --git a/packages/core/src/test/codewhisperer/startSecurityScan.test.ts b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts index 718769eaa22..bf05395f48d 100644 --- a/packages/core/src/test/codewhisperer/startSecurityScan.test.ts +++ b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts @@ -28,7 +28,7 @@ import * as model from '../../codewhisperer/models/model' import { CodewhispererSecurityScan } from '../../shared/telemetry/telemetry.gen' import * as errors from '../../shared/errors' import * as timeoutUtils from '../../shared/utilities/timeoutUtils' -import { createMockClient, mockGetCodeScanResponse } from './utils' +import { createMockClient, mockGetCodeScanResponse } from './testUtil' let extensionContext: FakeExtensionContext let mockSecurityPanelViewProvider: SecurityPanelViewProvider diff --git a/packages/core/src/test/codewhisperer/testUtil.ts b/packages/core/src/test/codewhisperer/testUtil.ts index 217837d33ed..6f6692279d1 100644 --- a/packages/core/src/test/codewhisperer/testUtil.ts +++ b/packages/core/src/test/codewhisperer/testUtil.ts @@ -20,11 +20,13 @@ import { session } from '../../codewhisperer/util/codeWhispererSession' import fs from 'fs' import { DefaultAWSClientBuilder, ServiceOptions } from '../../shared/awsClientBuilder' import { FakeAwsContext } from '../utilities/fakeAwsContext' -import { Service } from 'aws-sdk' +import { HttpResponse, Service } from 'aws-sdk' import userApiConfig = require('./../../codewhisperer/client/user-service-2.json') import CodeWhispererUserClient = require('../../codewhisperer/client/codewhispereruserclient') import { codeWhispererClient } from '../../codewhisperer/client/codewhisperer' import { RecommendationHandler } from '../../codewhisperer/service/recommendationHandler' +import * as model from '../../codewhisperer/models/model' +import { stub } from '../utilities/stubber' export async function resetCodeWhispererGlobalVariables() { vsCodeState.isIntelliSenseActive = false @@ -207,3 +209,106 @@ export function createMockDirentFile(fileName: string): fs.Dirent { dirent.name = fileName return dirent } + +export const mockGetCodeScanResponse = { + $response: { + data: { + status: 'Completed', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + status: 'Completed', +} + +export function createMockClient() { + const mockClient = stub(codewhispererClient.DefaultCodeWhispererClient) + + const mockCreateCodeScanResponse = { + $response: { + data: { + jobId: 'jobId', + status: 'Pending', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + jobId: 'jobId', + status: 'Pending', + } + const mockCreateUploadUrlResponse = { + $response: { + data: { + uploadId: 'uploadId', + uploadUrl: 'uploadUrl', + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + uploadId: 'uploadId', + uploadUrl: 'https://test.com', + } + + const mockCodeScanFindings = JSON.stringify([ + { + filePath: 'workspaceFolder/python3.7-plain-sam-app/hello_world/app.py', + startLine: 1, + endLine: 1, + title: 'title', + description: { + text: 'text', + markdown: 'markdown', + }, + detectorId: 'detectorId', + detectorName: 'detectorName', + findingId: 'findingId', + relatedVulnerabilities: [], + severity: 'High', + remediation: { + recommendation: { + text: 'text', + url: 'url', + }, + suggestedFixes: [], + }, + codeSnippet: [], + } satisfies model.RawCodeScanIssue, + ]) + + const mockListCodeScanFindingsResponse = { + $response: { + data: { + codeScanFindings: mockCodeScanFindings, + }, + requestId: 'requestId', + hasNextPage: () => false, + error: undefined, + nextPage: () => undefined, + redirectCount: 0, + retryCount: 0, + httpResponse: new HttpResponse(), + }, + codeScanFindings: mockCodeScanFindings, + } + + mockClient.createCodeScan.resolves(mockCreateCodeScanResponse) + mockClient.createUploadUrl.resolves(mockCreateUploadUrlResponse) + mockClient.getCodeScan.resolves(mockGetCodeScanResponse) + mockClient.listCodeScanFindings.resolves(mockListCodeScanFindingsResponse) + return mockClient +} From 56819a5e0359dcf11c3b80dea4955ecf16bb007d Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 8 Oct 2024 17:42:45 -0400 Subject: [PATCH 11/27] fix tests again --- .../codewhisperer/startSecurityScan.test.ts | 28 +++++++++---------- .../core/src/test/codewhisperer/testUtil.ts | 2 +- .../src/testInteg/startSecurityScan.test.ts | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/core/src/test/codewhisperer/startSecurityScan.test.ts b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts index bf05395f48d..fadd1063aa8 100644 --- a/packages/core/src/test/codewhisperer/startSecurityScan.test.ts +++ b/packages/core/src/test/codewhisperer/startSecurityScan.test.ts @@ -28,7 +28,7 @@ import * as model from '../../codewhisperer/models/model' import { CodewhispererSecurityScan } from '../../shared/telemetry/telemetry.gen' import * as errors from '../../shared/errors' import * as timeoutUtils from '../../shared/utilities/timeoutUtils' -import { createMockClient, mockGetCodeScanResponse } from './testUtil' +import { createClient, mockGetCodeScanResponse } from './testUtil' let extensionContext: FakeExtensionContext let mockSecurityPanelViewProvider: SecurityPanelViewProvider @@ -69,7 +69,7 @@ describe('startSecurityScan', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -88,7 +88,7 @@ describe('startSecurityScan', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -116,7 +116,7 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -142,7 +142,7 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -162,7 +162,7 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -186,7 +186,7 @@ describe('startSecurityScan', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.PROJECT ) @@ -205,14 +205,14 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.FILE ) await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -236,14 +236,14 @@ describe('startSecurityScan', function () { const scanPromise = startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.PROJECT ) await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.FILE ) @@ -263,7 +263,7 @@ describe('startSecurityScan', function () { it('Should handle failed scan job status', async function () { getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const mockClient = createMockClient() + const mockClient = createClient() mockClient.getCodeScan.resolves({ ...mockGetCodeScanResponse, status: 'Failed', @@ -286,7 +286,7 @@ describe('startSecurityScan', function () { it('Should show notification when throttled for project scans', async function () { getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const mockClient = createMockClient() + const mockClient = createClient() mockClient.createCodeScan.throws({ code: 'ThrottlingException', time: new Date(), @@ -315,7 +315,7 @@ describe('startSecurityScan', function () { it('Should set monthly quota exceeded when throttled for file scans', async function () { getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) await model.CodeScansState.instance.setScansEnabled(true) - const mockClient = createMockClient() + const mockClient = createClient() mockClient.createCodeScan.throws({ code: 'ThrottlingException', time: new Date(), diff --git a/packages/core/src/test/codewhisperer/testUtil.ts b/packages/core/src/test/codewhisperer/testUtil.ts index 6f6692279d1..d9a0672b86a 100644 --- a/packages/core/src/test/codewhisperer/testUtil.ts +++ b/packages/core/src/test/codewhisperer/testUtil.ts @@ -226,7 +226,7 @@ export const mockGetCodeScanResponse = { status: 'Completed', } -export function createMockClient() { +export function createClient() { const mockClient = stub(codewhispererClient.DefaultCodeWhispererClient) const mockCreateCodeScanResponse = { diff --git a/packages/core/src/testInteg/startSecurityScan.test.ts b/packages/core/src/testInteg/startSecurityScan.test.ts index d3a17a1650a..ea83c53651e 100644 --- a/packages/core/src/testInteg/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/startSecurityScan.test.ts @@ -23,7 +23,7 @@ import { getTestWindow } from '../test/shared/vscode/window' import { SeverityLevel } from '../test/shared/vscode/message' import { CodeAnalysisScope } from '../codewhisperer' import { performanceTest } from '../shared/performance/performance' -import { createMockClient } from '../test/amazonqFeatureDev/utils' +import { createClient } from '../test/codewhisperer/testUtil' describe('startSecurityScanPerformanceTest', function () { let extensionContext: FakeExtensionContext @@ -70,7 +70,7 @@ describe('startSecurityScanPerformanceTest', function () { await startSecurityScan.startSecurityScan( mockSecurityPanelViewProvider, editor, - createMockClient(), + createClient(), extensionContext, CodeAnalysisScope.FILE ) From da6c8ce7fff3da82f02e4d71b3b16aa50864f4ad Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 10 Oct 2024 15:51:06 -0400 Subject: [PATCH 12/27] initial work --- .../core/src/amazonqFeatureDev/util/files.ts | 7 +-- .../core/src/shared/performance/zipSpy.ts | 23 ++++++++++ .../amazonqFeatureDev/prepareRepoData.test.ts | 46 +++++++++++++++---- 3 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 packages/core/src/shared/performance/zipSpy.ts diff --git a/packages/core/src/amazonqFeatureDev/util/files.ts b/packages/core/src/amazonqFeatureDev/util/files.ts index cffa74af867..1b83bdbe2b5 100644 --- a/packages/core/src/amazonqFeatureDev/util/files.ts +++ b/packages/core/src/amazonqFeatureDev/util/files.ts @@ -18,6 +18,7 @@ import { AmazonqCreateUpload, Span, telemetry as amznTelemetry } from '../../sha import { TelemetryHelper } from './telemetryHelper' import { maxRepoSizeBytes } from '../constants' import { isCodeFile } from '../../shared/filetypes' +import { fs } from '../../shared' const getSha256 = (file: Buffer) => createHash('sha256').update(file).digest('base64') @@ -28,17 +29,17 @@ export async function prepareRepoData( repoRootPaths: string[], workspaceFolders: CurrentWsFolders, telemetry: TelemetryHelper, - span: Span + span: Span, + zip: AdmZip = new AdmZip() ) { try { const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes) - const zip = new AdmZip() let totalBytes = 0 const ignoredExtensionMap = new Map() for (const file of files) { - const fileSize = (await vscode.workspace.fs.stat(file.fileUri)).size + const fileSize = (await fs.stat(file.fileUri)).size const isCodeFile_ = isCodeFile(file.relativeFilePath) if (fileSize >= maxFileSizeBytes || !isCodeFile_) { diff --git a/packages/core/src/shared/performance/zipSpy.ts b/packages/core/src/shared/performance/zipSpy.ts new file mode 100644 index 00000000000..b8c1aab9c29 --- /dev/null +++ b/packages/core/src/shared/performance/zipSpy.ts @@ -0,0 +1,23 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import AdmZip from 'adm-zip' +import * as sinon from 'sinon' + +type AdmZipMethod = 'addLocalFile' +type AdmZipMethodMap = Record> + +export class AdmZipSpy { + private methods: AdmZipMethodMap + public constructor(instance: AdmZip) { + this.methods = { + addLocalFile: sinon.spy(instance, 'addLocalFile'), + } + } + + public get addLocalFile(): sinon.SinonSpy { + return this.methods.addLocalFile + } +} diff --git a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts index 50a4093ceca..83a0fe46e7e 100644 --- a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts +++ b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts @@ -3,22 +3,33 @@ * SPDX-License-Identifier: Apache-2.0 */ import assert from 'assert' +import * as sinon from 'sinon' import { WorkspaceFolder } from 'vscode' import { performanceTest } from '../../shared/performance/performance' import { createTestWorkspace } from '../testUtil' import { prepareRepoData, TelemetryHelper } from '../../amazonqFeatureDev' -import { AmazonqCreateUpload, getRandomString } from '../../shared' +import { AmazonqCreateUpload, fs, getRandomString } from '../../shared' import { Span } from '../../shared/telemetry' +import { FileSystem } from '../../shared/fs/fs' +import { AdmZipSpy } from '../../shared/performance/zipSpy' +import AdmZip from 'adm-zip' type resultType = { zipFileBuffer: Buffer zipFileChecksum: string } +type setupResult = { + workspace: WorkspaceFolder + initialZip: AdmZip + fsSpy: sinon.SinonSpiedInstance + zipSpy: AdmZipSpy +} + function performanceTestWrapper(numFiles: number, fileSize: number) { return performanceTest( { - testRuns: 10, + testRuns: 1, linux: { userCpuUsage: 100, systemCpuUsage: 35, @@ -40,34 +51,49 @@ function performanceTestWrapper(numFiles: number, fileSize: number) { const telemetry = new TelemetryHelper() return { setup: async () => { - return await createTestWorkspace(numFiles, { + const initialZip = new AdmZip() + const fsSpy = sinon.spy(fs) + const zipSpy = new AdmZipSpy(initialZip) + const workspace = await createTestWorkspace(numFiles, { fileNamePrefix: 'file', fileContent: getRandomString(fileSize), fileNameSuffix: '.md', }) + return { workspace, initialZip, fsSpy, zipSpy } }, - execute: async (workspace: WorkspaceFolder) => { - return await prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, { - record: () => {}, - } as unknown as Span) + execute: async (setup: setupResult) => { + return await prepareRepoData( + [setup.workspace.uri.fsPath], + [setup.workspace], + telemetry, + { + record: () => {}, + } as unknown as Span, + setup.initialZip + ) }, - verify: async (_w: WorkspaceFolder, result: resultType) => { - verifyResult(result, telemetry, numFiles * fileSize) + verify: async (setup: setupResult, result: resultType) => { + verifyResult(setup, result, telemetry, numFiles * fileSize) }, } } ) } -function verifyResult(result: resultType, telemetry: TelemetryHelper, expectedSize: number): void { +function verifyResult(setup: setupResult, result: resultType, telemetry: TelemetryHelper, expectedSize: number): void { assert.ok(result) assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true) assert.strictEqual(telemetry.repositorySize, expectedSize) assert.strictEqual(result.zipFileChecksum.length, 44) + assert.ok(setup.fsSpy.appendFile.callCount === 0) + assert.ok(setup.zipSpy.addLocalFile.callCount > 0) } describe('prepareRepoData', function () { describe('Performance Tests', function () { + afterEach(function () { + sinon.restore() + }) performanceTestWrapper(250, 10) performanceTestWrapper(10, 1000) }) From b3619a97c2ef8133e01ccee9af1e6025c4a80d98 Mon Sep 17 00:00:00 2001 From: hkobew Date: Thu, 10 Oct 2024 15:56:17 -0400 Subject: [PATCH 13/27] delete unneeded code --- .../core/src/shared/performance/zipSpy.ts | 23 ------------------- .../amazonqFeatureDev/prepareRepoData.test.ts | 7 +++--- 2 files changed, 3 insertions(+), 27 deletions(-) delete mode 100644 packages/core/src/shared/performance/zipSpy.ts diff --git a/packages/core/src/shared/performance/zipSpy.ts b/packages/core/src/shared/performance/zipSpy.ts deleted file mode 100644 index b8c1aab9c29..00000000000 --- a/packages/core/src/shared/performance/zipSpy.ts +++ /dev/null @@ -1,23 +0,0 @@ -/*! - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -import AdmZip from 'adm-zip' -import * as sinon from 'sinon' - -type AdmZipMethod = 'addLocalFile' -type AdmZipMethodMap = Record> - -export class AdmZipSpy { - private methods: AdmZipMethodMap - public constructor(instance: AdmZip) { - this.methods = { - addLocalFile: sinon.spy(instance, 'addLocalFile'), - } - } - - public get addLocalFile(): sinon.SinonSpy { - return this.methods.addLocalFile - } -} diff --git a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts index 83a0fe46e7e..127630f9643 100644 --- a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts +++ b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts @@ -11,7 +11,6 @@ import { prepareRepoData, TelemetryHelper } from '../../amazonqFeatureDev' import { AmazonqCreateUpload, fs, getRandomString } from '../../shared' import { Span } from '../../shared/telemetry' import { FileSystem } from '../../shared/fs/fs' -import { AdmZipSpy } from '../../shared/performance/zipSpy' import AdmZip from 'adm-zip' type resultType = { @@ -23,7 +22,7 @@ type setupResult = { workspace: WorkspaceFolder initialZip: AdmZip fsSpy: sinon.SinonSpiedInstance - zipSpy: AdmZipSpy + zipSpy: sinon.SinonSpiedInstance } function performanceTestWrapper(numFiles: number, fileSize: number) { @@ -53,7 +52,7 @@ function performanceTestWrapper(numFiles: number, fileSize: number) { setup: async () => { const initialZip = new AdmZip() const fsSpy = sinon.spy(fs) - const zipSpy = new AdmZipSpy(initialZip) + const zipSpy = sinon.spy(initialZip) const workspace = await createTestWorkspace(numFiles, { fileNamePrefix: 'file', fileContent: getRandomString(fileSize), @@ -85,7 +84,7 @@ function verifyResult(setup: setupResult, result: resultType, telemetry: Telemet assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true) assert.strictEqual(telemetry.repositorySize, expectedSize) assert.strictEqual(result.zipFileChecksum.length, 44) - assert.ok(setup.fsSpy.appendFile.callCount === 0) + assert.ok(setup.fsSpy.stat.callCount > 0) assert.ok(setup.zipSpy.addLocalFile.callCount > 0) } From 685924cf43926e5e4ff2b3b390dc784344772c61 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 11:19:48 -0400 Subject: [PATCH 14/27] move tests into testPerf --- .../shared/utilities/workspaceUtils.test.ts | 52 ---------------- .../{ => testPerf}/buildIndex.test.ts | 10 +-- .../testInteg/testPerf/collectFiles.test.ts | 62 +++++++++++++++++++ .../prepareRepoData.test.ts | 0 .../{ => testPerf}/startSecurityScan.test.ts | 24 +++---- .../{ => testPerf}/tryInstallLsp.test.ts | 8 +-- .../testInteg/{ => testPerf}/zipcode.test.ts | 12 ++-- 7 files changed, 89 insertions(+), 79 deletions(-) rename packages/core/src/testInteg/{ => testPerf}/buildIndex.test.ts (87%) create mode 100644 packages/core/src/testInteg/testPerf/collectFiles.test.ts rename packages/core/src/testInteg/{amazonqFeatureDev => testPerf}/prepareRepoData.test.ts (100%) rename packages/core/src/testInteg/{ => testPerf}/startSecurityScan.test.ts (79%) rename packages/core/src/testInteg/{ => testPerf}/tryInstallLsp.test.ts (94%) rename packages/core/src/testInteg/{ => testPerf}/zipcode.test.ts (87%) diff --git a/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts b/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts index 19dd0d965a1..95619f0599c 100644 --- a/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts +++ b/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts @@ -18,8 +18,6 @@ import globals from '../../../shared/extensionGlobals' import { CodelensRootRegistry } from '../../../shared/fs/codelensRootRegistry' import { createTestWorkspace, createTestWorkspaceFolder, toFile } from '../../../test/testUtil' import sinon from 'sinon' -import { performanceTest } from '../../../shared/performance/performance' -import { randomUUID } from '../../../shared/crypto' import { fs } from '../../../shared' describe('findParentProjectFile', async function () { @@ -326,56 +324,6 @@ describe('collectFiles', function () { assert.deepStrictEqual(1, result.length) assert.deepStrictEqual('non-license.md', result[0].relativeFilePath) }) - - performanceTest( - // collecting all files in the workspace and zipping them is pretty resource intensive - { - linux: { - userCpuUsage: 85, - heapTotal: 2, - duration: 0.8, - }, - }, - 'calculate cpu and memory usage', - function () { - const totalFiles = 100 - return { - setup: async () => { - const workspace = await createTestWorkspaceFolder() - - sinon.stub(vscode.workspace, 'workspaceFolders').value([workspace]) - - const fileContent = randomUUID() - for (let x = 0; x < totalFiles; x++) { - await toFile(fileContent, path.join(workspace.uri.fsPath, `file.${x}`)) - } - - return { - workspace, - } - }, - execute: async ({ workspace }: { workspace: vscode.WorkspaceFolder }) => { - return { - result: await collectFiles([workspace.uri.fsPath], [workspace], true), - } - }, - verify: ( - _: { workspace: vscode.WorkspaceFolder }, - { result }: { result: Awaited> } - ) => { - assert.deepStrictEqual(result.length, totalFiles) - const sortedFiles = [...result].sort((a, b) => { - const numA = parseInt(a.relativeFilePath.split('.')[1]) - const numB = parseInt(b.relativeFilePath.split('.')[1]) - return numA - numB - }) - for (let x = 0; x < totalFiles; x++) { - assert.deepStrictEqual(sortedFiles[x].relativeFilePath, `file.${x}`) - } - }, - } - } - ) }) describe('getWorkspaceFoldersByPrefixes', function () { diff --git a/packages/core/src/testInteg/buildIndex.test.ts b/packages/core/src/testInteg/testPerf/buildIndex.test.ts similarity index 87% rename from packages/core/src/testInteg/buildIndex.test.ts rename to packages/core/src/testInteg/testPerf/buildIndex.test.ts index 261327e5145..29f07e28587 100644 --- a/packages/core/src/testInteg/buildIndex.test.ts +++ b/packages/core/src/testInteg/testPerf/buildIndex.test.ts @@ -3,15 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { performanceTest } from '../shared/performance/performance' +import { performanceTest } from '../../shared/performance/performance' import * as sinon from 'sinon' import * as vscode from 'vscode' import assert from 'assert' -import { LspClient, LspController } from '../amazonq' +import { LspClient, LspController } from '../../amazonq' import { LanguageClient, ServerOptions } from 'vscode-languageclient' -import { createTestWorkspace } from '../test/testUtil' -import { GetUsageRequestType, IndexRequestType } from '../amazonq/lsp/types' -import { getRandomString } from '../shared' +import { createTestWorkspace } from '../../test/testUtil' +import { GetUsageRequestType, IndexRequestType } from '../../amazonq/lsp/types' +import { getRandomString } from '../../shared' interface SetupResult { clientReqStub: sinon.SinonStub diff --git a/packages/core/src/testInteg/testPerf/collectFiles.test.ts b/packages/core/src/testInteg/testPerf/collectFiles.test.ts new file mode 100644 index 00000000000..3e9914898f5 --- /dev/null +++ b/packages/core/src/testInteg/testPerf/collectFiles.test.ts @@ -0,0 +1,62 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import assert from 'assert' +import * as vscode from 'vscode' +import * as sinon from 'sinon' +import { performanceTest } from '../../shared/performance/performance' +import { createTestWorkspaceFolder, toFile } from '../../test/testUtil' +import path from 'path' +import { randomUUID } from '../../shared' +import { collectFiles } from '../../shared/utilities/workspaceUtils' + +performanceTest( + // collecting all files in the workspace and zipping them is pretty resource intensive + { + linux: { + userCpuUsage: 85, + heapTotal: 2, + duration: 0.8, + }, + }, + 'calculate cpu and memory usage', + function () { + const totalFiles = 100 + return { + setup: async () => { + const workspace = await createTestWorkspaceFolder() + + sinon.stub(vscode.workspace, 'workspaceFolders').value([workspace]) + + const fileContent = randomUUID() + for (let x = 0; x < totalFiles; x++) { + await toFile(fileContent, path.join(workspace.uri.fsPath, `file.${x}`)) + } + + return { + workspace, + } + }, + execute: async ({ workspace }: { workspace: vscode.WorkspaceFolder }) => { + return { + result: await collectFiles([workspace.uri.fsPath], [workspace], true), + } + }, + verify: ( + _: { workspace: vscode.WorkspaceFolder }, + { result }: { result: Awaited> } + ) => { + assert.deepStrictEqual(result.length, totalFiles) + const sortedFiles = [...result].sort((a, b) => { + const numA = parseInt(a.relativeFilePath.split('.')[1]) + const numB = parseInt(b.relativeFilePath.split('.')[1]) + return numA - numB + }) + for (let x = 0; x < totalFiles; x++) { + assert.deepStrictEqual(sortedFiles[x].relativeFilePath, `file.${x}`) + } + }, + } + } +) diff --git a/packages/core/src/testInteg/amazonqFeatureDev/prepareRepoData.test.ts b/packages/core/src/testInteg/testPerf/prepareRepoData.test.ts similarity index 100% rename from packages/core/src/testInteg/amazonqFeatureDev/prepareRepoData.test.ts rename to packages/core/src/testInteg/testPerf/prepareRepoData.test.ts diff --git a/packages/core/src/testInteg/startSecurityScan.test.ts b/packages/core/src/testInteg/testPerf/startSecurityScan.test.ts similarity index 79% rename from packages/core/src/testInteg/startSecurityScan.test.ts rename to packages/core/src/testInteg/testPerf/startSecurityScan.test.ts index ea83c53651e..79209322d31 100644 --- a/packages/core/src/testInteg/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/testPerf/startSecurityScan.test.ts @@ -4,13 +4,13 @@ */ import * as vscode from 'vscode' import * as sinon from 'sinon' -import * as startSecurityScan from '../codewhisperer/commands/startSecurityScan' -import * as diagnosticsProvider from '../codewhisperer/service/diagnosticsProvider' -import * as model from '../codewhisperer/models/model' -import * as timeoutUtils from '../shared/utilities/timeoutUtils' +import * as startSecurityScan from '../../codewhisperer/commands/startSecurityScan' +import * as diagnosticsProvider from '../../codewhisperer/service/diagnosticsProvider' +import * as model from '../../codewhisperer/models/model' +import * as timeoutUtils from '../../shared/utilities/timeoutUtils' import assert from 'assert' -import { SecurityPanelViewProvider } from '../codewhisperer/views/securityPanelViewProvider' -import { FakeExtensionContext } from '../test/fakeExtensionContext' +import { SecurityPanelViewProvider } from '../../codewhisperer/views/securityPanelViewProvider' +import { FakeExtensionContext } from '../../test/fakeExtensionContext' import { join } from 'path' import { assertTelemetry, @@ -18,12 +18,12 @@ import { createTestWorkspaceFolder, getFetchStubWithResponse, toFile, -} from '../test/testUtil' -import { getTestWindow } from '../test/shared/vscode/window' -import { SeverityLevel } from '../test/shared/vscode/message' -import { CodeAnalysisScope } from '../codewhisperer' -import { performanceTest } from '../shared/performance/performance' -import { createClient } from '../test/codewhisperer/testUtil' +} from '../../test/testUtil' +import { getTestWindow } from '../../test/shared/vscode/window' +import { SeverityLevel } from '../../test/shared/vscode/message' +import { CodeAnalysisScope } from '../../codewhisperer' +import { performanceTest } from '../../shared/performance/performance' +import { createClient } from '../../test/codewhisperer/testUtil' describe('startSecurityScanPerformanceTest', function () { let extensionContext: FakeExtensionContext diff --git a/packages/core/src/testInteg/tryInstallLsp.test.ts b/packages/core/src/testInteg/testPerf/tryInstallLsp.test.ts similarity index 94% rename from packages/core/src/testInteg/tryInstallLsp.test.ts rename to packages/core/src/testInteg/testPerf/tryInstallLsp.test.ts index 9a5f8b808db..7cc9c18b2c4 100644 --- a/packages/core/src/testInteg/tryInstallLsp.test.ts +++ b/packages/core/src/testInteg/testPerf/tryInstallLsp.test.ts @@ -7,10 +7,10 @@ import sinon from 'sinon' import { Content } from 'aws-sdk/clients/codecommit' import AdmZip from 'adm-zip' import path from 'path' -import { LspController } from '../amazonq' -import { fs, getRandomString, globals } from '../shared' -import { createTestWorkspace } from '../test/testUtil' -import { performanceTest } from '../shared/performance/performance' +import { LspController } from '../../amazonq' +import { fs, getRandomString, globals } from '../../shared' +import { createTestWorkspace } from '../../test/testUtil' +import { performanceTest } from '../../shared/performance/performance' // fakeFileContent is matched to fakeQServerContent based on hash. const fakeHash = '4eb2865c8f40a322aa04e17d8d83bdaa605d6f1cb363af615240a5442a010e0aef66e21bcf4c88f20fabff06efe8a214' diff --git a/packages/core/src/testInteg/zipcode.test.ts b/packages/core/src/testInteg/testPerf/zipcode.test.ts similarity index 87% rename from packages/core/src/testInteg/zipcode.test.ts rename to packages/core/src/testInteg/testPerf/zipcode.test.ts index 685c26c02e8..1883861f222 100644 --- a/packages/core/src/testInteg/zipcode.test.ts +++ b/packages/core/src/testInteg/testPerf/zipcode.test.ts @@ -4,12 +4,12 @@ */ import assert from 'assert' import * as sinon from 'sinon' -import { TransformByQState, ZipManifest } from '../codewhisperer' -import { fs, getRandomString, globals } from '../shared' -import { createTestWorkspace } from '../test/testUtil' -import * as CodeWhispererConstants from '../codewhisperer/models/constants' -import { performanceTest } from '../shared/performance/performance' -import { zipCode } from '../codewhisperer/indexNode' +import { TransformByQState, ZipManifest } from '../../codewhisperer' +import { fs, getRandomString, globals } from '../../shared' +import { createTestWorkspace } from '../../test/testUtil' +import * as CodeWhispererConstants from '../../codewhisperer/models/constants' +import { performanceTest } from '../../shared/performance/performance' +import { zipCode } from '../../codewhisperer/indexNode' interface SetupResult { tempDir: string From 0aa5546b2a7afa08b809ce1220588d8dfd901973 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 11:25:12 -0400 Subject: [PATCH 15/27] rename folder --- packages/core/src/testInteg/{testPerf => perf}/buildIndex.test.ts | 0 .../core/src/testInteg/{testPerf => perf}/collectFiles.test.ts | 0 .../core/src/testInteg/{testPerf => perf}/prepareRepoData.test.ts | 0 .../src/testInteg/{testPerf => perf}/startSecurityScan.test.ts | 0 .../core/src/testInteg/{testPerf => perf}/tryInstallLsp.test.ts | 0 packages/core/src/testInteg/{testPerf => perf}/zipcode.test.ts | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename packages/core/src/testInteg/{testPerf => perf}/buildIndex.test.ts (100%) rename packages/core/src/testInteg/{testPerf => perf}/collectFiles.test.ts (100%) rename packages/core/src/testInteg/{testPerf => perf}/prepareRepoData.test.ts (100%) rename packages/core/src/testInteg/{testPerf => perf}/startSecurityScan.test.ts (100%) rename packages/core/src/testInteg/{testPerf => perf}/tryInstallLsp.test.ts (100%) rename packages/core/src/testInteg/{testPerf => perf}/zipcode.test.ts (100%) diff --git a/packages/core/src/testInteg/testPerf/buildIndex.test.ts b/packages/core/src/testInteg/perf/buildIndex.test.ts similarity index 100% rename from packages/core/src/testInteg/testPerf/buildIndex.test.ts rename to packages/core/src/testInteg/perf/buildIndex.test.ts diff --git a/packages/core/src/testInteg/testPerf/collectFiles.test.ts b/packages/core/src/testInteg/perf/collectFiles.test.ts similarity index 100% rename from packages/core/src/testInteg/testPerf/collectFiles.test.ts rename to packages/core/src/testInteg/perf/collectFiles.test.ts diff --git a/packages/core/src/testInteg/testPerf/prepareRepoData.test.ts b/packages/core/src/testInteg/perf/prepareRepoData.test.ts similarity index 100% rename from packages/core/src/testInteg/testPerf/prepareRepoData.test.ts rename to packages/core/src/testInteg/perf/prepareRepoData.test.ts diff --git a/packages/core/src/testInteg/testPerf/startSecurityScan.test.ts b/packages/core/src/testInteg/perf/startSecurityScan.test.ts similarity index 100% rename from packages/core/src/testInteg/testPerf/startSecurityScan.test.ts rename to packages/core/src/testInteg/perf/startSecurityScan.test.ts diff --git a/packages/core/src/testInteg/testPerf/tryInstallLsp.test.ts b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts similarity index 100% rename from packages/core/src/testInteg/testPerf/tryInstallLsp.test.ts rename to packages/core/src/testInteg/perf/tryInstallLsp.test.ts diff --git a/packages/core/src/testInteg/testPerf/zipcode.test.ts b/packages/core/src/testInteg/perf/zipcode.test.ts similarity index 100% rename from packages/core/src/testInteg/testPerf/zipcode.test.ts rename to packages/core/src/testInteg/perf/zipcode.test.ts From b0b63ad88397de6b1370702a2c891acce5e6dc4d Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 11:56:47 -0400 Subject: [PATCH 16/27] implement spy --- .../test/amazonqFeatureDev/prepareRepoData.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts index 127630f9643..95b88782ee1 100644 --- a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts +++ b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts @@ -23,6 +23,8 @@ type setupResult = { initialZip: AdmZip fsSpy: sinon.SinonSpiedInstance zipSpy: sinon.SinonSpiedInstance + numFiles: number + fileSize: number } function performanceTestWrapper(numFiles: number, fileSize: number) { @@ -58,7 +60,7 @@ function performanceTestWrapper(numFiles: number, fileSize: number) { fileContent: getRandomString(fileSize), fileNameSuffix: '.md', }) - return { workspace, initialZip, fsSpy, zipSpy } + return { workspace, initialZip, fsSpy, zipSpy, numFiles, fileSize } }, execute: async (setup: setupResult) => { return await prepareRepoData( @@ -84,8 +86,12 @@ function verifyResult(setup: setupResult, result: resultType, telemetry: Telemet assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true) assert.strictEqual(telemetry.repositorySize, expectedSize) assert.strictEqual(result.zipFileChecksum.length, 44) - assert.ok(setup.fsSpy.stat.callCount > 0) - assert.ok(setup.zipSpy.addLocalFile.callCount > 0) + + assert.ok(setup.fsSpy.stat.callCount <= setup.numFiles * 2, 'calls stat at most twice per file') + // Sometimes this line fails locally when it finds additional. This shouldn't happen in CI. + assert.ok(setup.fsSpy.readFileText.callCount <= setup.numFiles, 'reads each file at most once') + assert.ok(setup.zipSpy.addLocalFile.callCount <= setup.numFiles, 'add files to zip at most once') + assert.strictEqual(setup.zipSpy.toBuffer.callCount, 1, 'creates buffer once') } describe('prepareRepoData', function () { From 4fe7c7b1b67aab54ac7df75094703e30efebcb94 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 11:57:51 -0400 Subject: [PATCH 17/27] increase thresholds --- .../core/src/test/amazonqFeatureDev/prepareRepoData.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts index 95b88782ee1..f285b90dae0 100644 --- a/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts +++ b/packages/core/src/test/amazonqFeatureDev/prepareRepoData.test.ts @@ -32,17 +32,17 @@ function performanceTestWrapper(numFiles: number, fileSize: number) { { testRuns: 1, linux: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 4, }, darwin: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 4, }, win32: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 4, }, From fda96df0b66ee16bd1c680ea391f108ef2bcf25d Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 14:04:38 -0400 Subject: [PATCH 18/27] build shared utility --- .../src/shared/utilities/workspaceUtils.ts | 2 +- .../src/testInteg/perf/buildIndex.test.ts | 14 ++++- .../testInteg/perf/prepareRepoData.test.ts | 12 ++-- packages/core/src/testInteg/perf/utilities.ts | 60 +++++++++++++++++++ 4 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 packages/core/src/testInteg/perf/utilities.ts diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts index f7bea42beaa..d8bd9177592 100644 --- a/packages/core/src/shared/utilities/workspaceUtils.ts +++ b/packages/core/src/shared/utilities/workspaceUtils.ts @@ -582,7 +582,7 @@ export async function collectFilesForIndex( continue } - const fileStat = await vscode.workspace.fs.stat(file) + const fileStat = await fs.stat(file) // ignore single file over 10 MB if (fileStat.size > 10 * 1024 * 1024) { continue diff --git a/packages/core/src/testInteg/perf/buildIndex.test.ts b/packages/core/src/testInteg/perf/buildIndex.test.ts index 29f07e28587..721a8c19438 100644 --- a/packages/core/src/testInteg/perf/buildIndex.test.ts +++ b/packages/core/src/testInteg/perf/buildIndex.test.ts @@ -11,26 +11,36 @@ import { LspClient, LspController } from '../../amazonq' import { LanguageClient, ServerOptions } from 'vscode-languageclient' import { createTestWorkspace } from '../../test/testUtil' import { GetUsageRequestType, IndexRequestType } from '../../amazonq/lsp/types' -import { getRandomString } from '../../shared' +import { fs, getRandomString } from '../../shared' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' interface SetupResult { clientReqStub: sinon.SinonStub + fsSpy: sinon.SinonSpiedInstance + findFilesSpy: sinon.SinonSpy } async function verifyResult(setup: SetupResult) { assert.ok(setup.clientReqStub.calledTwice) assert.ok(setup.clientReqStub.firstCall.calledWith(IndexRequestType)) assert.ok(setup.clientReqStub.secondCall.calledWith(GetUsageRequestType)) + + assert.strictEqual(getFsCallsUpperBound(setup.fsSpy), 0, 'should not make any fs calls') + assert.strictEqual(setup.findFilesSpy.callCount, 2, 'only make 2 calls to find all files in workspace') } async function setupWithWorkspace(numFiles: number, options: { fileContent: string }): Promise { // Force VSCode to find my test workspace only to keep test contained and controlled. const testWorksapce = await createTestWorkspace(numFiles, options) sinon.stub(vscode.workspace, 'workspaceFolders').value([testWorksapce]) + // Avoid sending real request to lsp. const clientReqStub = sinon.stub(LanguageClient.prototype, 'sendRequest').resolves(true) + const fsSpy = sinon.spy(fs) + const findFilesSpy = sinon.spy(vscode.workspace, 'findFiles') LspClient.instance.client = new LanguageClient('amazonq', 'test-client', {} as ServerOptions, {}) - return { clientReqStub } + return { clientReqStub, fsSpy, findFilesSpy } } describe('buildIndex', function () { diff --git a/packages/core/src/testInteg/perf/prepareRepoData.test.ts b/packages/core/src/testInteg/perf/prepareRepoData.test.ts index 28f20b65df4..58b599f51a2 100644 --- a/packages/core/src/testInteg/perf/prepareRepoData.test.ts +++ b/packages/core/src/testInteg/perf/prepareRepoData.test.ts @@ -12,6 +12,7 @@ import { AmazonqCreateUpload, fs, getRandomString } from '../../shared' import { Span } from '../../shared/telemetry' import { FileSystem } from '../../shared/fs/fs' import AdmZip from 'adm-zip' +import { assertEfficientZip, getFsCallsUpperBound } from './utilities' type resultType = { zipFileBuffer: Buffer @@ -87,11 +88,8 @@ function verifyResult(setup: setupResult, result: resultType, telemetry: Telemet assert.strictEqual(telemetry.repositorySize, expectedSize) assert.strictEqual(result.zipFileChecksum.length, 44) - assert.ok(setup.fsSpy.stat.callCount <= setup.numFiles * 2, 'calls stat at most twice per file') - // Sometimes this line fails locally when it finds additional. This shouldn't happen in CI. - assert.ok(setup.fsSpy.readFileText.callCount <= setup.numFiles, 'reads each file at most once') - assert.ok(setup.zipSpy.addLocalFile.callCount <= setup.numFiles, 'add files to zip at most once') - assert.strictEqual(setup.zipSpy.toBuffer.callCount, 1, 'creates buffer once') + assert.ok(getFsCallsUpperBound(setup.fsSpy) <= setup.numFiles * 4, 'total system calls should be under 4 per file') + assertEfficientZip(setup.zipSpy, setup.numFiles) } describe('prepareRepoData', function () { @@ -99,7 +97,9 @@ describe('prepareRepoData', function () { afterEach(function () { sinon.restore() }) - performanceTestWrapper(250, 10) performanceTestWrapper(10, 1000) + performanceTestWrapper(50, 500) + performanceTestWrapper(100, 100) + performanceTestWrapper(250, 10) }) }) diff --git a/packages/core/src/testInteg/perf/utilities.ts b/packages/core/src/testInteg/perf/utilities.ts new file mode 100644 index 00000000000..86ff69cc9c6 --- /dev/null +++ b/packages/core/src/testInteg/perf/utilities.ts @@ -0,0 +1,60 @@ +/*! + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import assert from 'assert' +import * as sinon from 'sinon' +import { FileSystem } from '../../shared/fs/fs' +import AdmZip from 'adm-zip' + +/** + * Provide an upper bound on total number of system calls done through our fs module. + * @param fsSpy filesystem spy (Ex. `sinon.spy(fs)`) + * @returns count of operations + */ +export function getFsCallsUpperBound(fsSpy: sinon.SinonSpiedInstance): number { + return getFsReadsUpperBound(fsSpy) + getFsWritesUpperBound(fsSpy) +} + +/** + * Provide an upper bound on the number of filesystem reads done through our fs module. + * This is an upper bound because some of the functions call eachother. + * @param fsSpy filesystem spy (Ex. `sinon.spy(fs)`) + * @returns value + */ +export function getFsReadsUpperBound(fsSpy: sinon.SinonSpiedInstance): number { + return ( + fsSpy.readFileBytes.callCount + + fsSpy.exists.callCount + + fsSpy.exists.callCount + + fsSpy.readdir.callCount + + fsSpy.copy.callCount + + fsSpy.checkPerms.callCount + + fsSpy.tryGetFilepathEnvVar.callCount + ) +} +/** + * Provide an upper bound on the number of filesystem writes done through our fs module. + * This is an upper bound because some of the functions call eachother. + * @param fsSpy filesystem spy (Ex. `sinon.spy(fs)`) + * @returns value + */ +export function getFsWritesUpperBound(fsSpy: sinon.SinonSpiedInstance): number { + return ( + fsSpy.writeFile.callCount + + fsSpy.mkdir.callCount + + fsSpy.rename.callCount + + fsSpy.chmod.callCount + + fsSpy.delete.callCount + + fsSpy.copy.callCount + ) +} +/** + * Check that each file is added to zip once, and written to buffer done exactly once. + * @param zipSpy spied AdmZip instance used by code. + * @param numFiles number of files in the workspace expected to be zipped. + */ +export function assertEfficientZip(zipSpy: sinon.SinonSpiedInstance, numFiles: number): void | never { + assert.ok(zipSpy.addLocalFile.callCount <= numFiles, 'add files to zip at most once') + assert.strictEqual(zipSpy.toBuffer.callCount, 1, 'creates buffer once') +} From 10130a67b5058bdda7b5bb68b3bb45fdf7145b4d Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 14:18:40 -0400 Subject: [PATCH 19/27] refactor collectFiles --- .../src/testInteg/perf/buildIndex.test.ts | 2 +- .../src/testInteg/perf/collectFiles.test.ts | 124 +++++++++++------- 2 files changed, 80 insertions(+), 46 deletions(-) diff --git a/packages/core/src/testInteg/perf/buildIndex.test.ts b/packages/core/src/testInteg/perf/buildIndex.test.ts index 721a8c19438..3b13746ab68 100644 --- a/packages/core/src/testInteg/perf/buildIndex.test.ts +++ b/packages/core/src/testInteg/perf/buildIndex.test.ts @@ -27,7 +27,7 @@ async function verifyResult(setup: SetupResult) { assert.ok(setup.clientReqStub.secondCall.calledWith(GetUsageRequestType)) assert.strictEqual(getFsCallsUpperBound(setup.fsSpy), 0, 'should not make any fs calls') - assert.strictEqual(setup.findFilesSpy.callCount, 2, 'only make 2 calls to find all files in workspace') + assert.ok(setup.findFilesSpy.callCount <= 2, 'findFiles should not be called more than twice') } async function setupWithWorkspace(numFiles: number, options: { fileContent: string }): Promise { diff --git a/packages/core/src/testInteg/perf/collectFiles.test.ts b/packages/core/src/testInteg/perf/collectFiles.test.ts index 3e9914898f5..79fc98bff2b 100644 --- a/packages/core/src/testInteg/perf/collectFiles.test.ts +++ b/packages/core/src/testInteg/perf/collectFiles.test.ts @@ -8,55 +8,89 @@ import * as sinon from 'sinon' import { performanceTest } from '../../shared/performance/performance' import { createTestWorkspaceFolder, toFile } from '../../test/testUtil' import path from 'path' -import { randomUUID } from '../../shared' +import { fs, randomUUID } from '../../shared' import { collectFiles } from '../../shared/utilities/workspaceUtils' +import { getFsCallsUpperBound } from './utilities' +import { FileSystem } from '../../shared/fs/fs' -performanceTest( - // collecting all files in the workspace and zipping them is pretty resource intensive - { - linux: { - userCpuUsage: 85, - heapTotal: 2, - duration: 0.8, +function performanceTestWrapper(totalFiles: number) { + return performanceTest( + { + darwin: { + userCpuUsage: 100, + systemCpuUsage: 35, + heapTotal: 2, + }, + linux: { + userCpuUsage: 100, + systemCpuUsage: 35, + heapTotal: 2, + }, + win32: { + userCpuUsage: 100, + systemCpuUsage: 35, + heapTotal: 2, + }, }, - }, - 'calculate cpu and memory usage', - function () { - const totalFiles = 100 - return { - setup: async () => { - const workspace = await createTestWorkspaceFolder() + 'calculate cpu and memory usage', + function () { + return { + setup: async () => { + const workspace = await createTestWorkspaceFolder() - sinon.stub(vscode.workspace, 'workspaceFolders').value([workspace]) + sinon.stub(vscode.workspace, 'workspaceFolders').value([workspace]) + const fsSpy = sinon.spy(fs) + const findFilesSpy = sinon.spy(vscode.workspace, 'findFiles') + const fileContent = randomUUID() + for (let x = 0; x < totalFiles; x++) { + await toFile(fileContent, path.join(workspace.uri.fsPath, `file.${x}`)) + } - const fileContent = randomUUID() - for (let x = 0; x < totalFiles; x++) { - await toFile(fileContent, path.join(workspace.uri.fsPath, `file.${x}`)) - } + return { + workspace, + fsSpy, + findFilesSpy, + } + }, + execute: async ({ workspace }: { workspace: vscode.WorkspaceFolder }) => { + return { + result: await collectFiles([workspace.uri.fsPath], [workspace], true), + } + }, + verify: ( + setup: { + workspace: vscode.WorkspaceFolder + fsSpy: sinon.SinonSpiedInstance + findFilesSpy: sinon.SinonSpy + }, + { result }: { result: Awaited> } + ) => { + assert.deepStrictEqual(result.length, totalFiles) + const sortedFiles = [...result].sort((a, b) => { + const numA = parseInt(a.relativeFilePath.split('.')[1]) + const numB = parseInt(b.relativeFilePath.split('.')[1]) + return numA - numB + }) + for (let x = 0; x < totalFiles; x++) { + assert.deepStrictEqual(sortedFiles[x].relativeFilePath, `file.${x}`) + } - return { - workspace, - } - }, - execute: async ({ workspace }: { workspace: vscode.WorkspaceFolder }) => { - return { - result: await collectFiles([workspace.uri.fsPath], [workspace], true), - } - }, - verify: ( - _: { workspace: vscode.WorkspaceFolder }, - { result }: { result: Awaited> } - ) => { - assert.deepStrictEqual(result.length, totalFiles) - const sortedFiles = [...result].sort((a, b) => { - const numA = parseInt(a.relativeFilePath.split('.')[1]) - const numB = parseInt(b.relativeFilePath.split('.')[1]) - return numA - numB - }) - for (let x = 0; x < totalFiles; x++) { - assert.deepStrictEqual(sortedFiles[x].relativeFilePath, `file.${x}`) - } - }, + assert.ok( + getFsCallsUpperBound(setup.fsSpy) <= totalFiles * 5, + 'total system calls below 5 per file' + ) + assert.ok(setup.findFilesSpy.callCount <= 2, 'findFiles not called more than twice') + }, + } } - } -) + ) +} + +describe('collectFiles', function () { + afterEach(function () { + sinon.restore() + }) + performanceTestWrapper(10) + performanceTestWrapper(100) + performanceTestWrapper(250) +}) From 55ee15e2b6bf94e96797abe35105b88acc2c3477 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 15:36:01 -0400 Subject: [PATCH 20/27] add spies for startSecScan --- .../commands/startSecurityScan.ts | 4 +- .../testInteg/perf/prepareRepoData.test.ts | 4 +- .../testInteg/perf/startSecurityScan.test.ts | 102 ++++++++++++------ packages/core/src/testInteg/perf/utilities.ts | 2 +- 4 files changed, 74 insertions(+), 38 deletions(-) diff --git a/packages/core/src/codewhisperer/commands/startSecurityScan.ts b/packages/core/src/codewhisperer/commands/startSecurityScan.ts index 8dee806f9df..58eaee2bf40 100644 --- a/packages/core/src/codewhisperer/commands/startSecurityScan.ts +++ b/packages/core/src/codewhisperer/commands/startSecurityScan.ts @@ -93,7 +93,8 @@ export async function startSecurityScan( editor: vscode.TextEditor | undefined, client: DefaultCodeWhispererClient, context: vscode.ExtensionContext, - scope: CodeWhispererConstants.CodeAnalysisScope + scope: CodeWhispererConstants.CodeAnalysisScope, + zipUtil: ZipUtil = new ZipUtil() ) { const logger = getLoggerForScope(scope) /** @@ -130,7 +131,6 @@ export async function startSecurityScan( * Step 1: Generate zip */ throwIfCancelled(scope, codeScanStartTime) - const zipUtil = new ZipUtil() const zipMetadata = await zipUtil.generateZip(editor?.document.uri, scope) const projectPaths = zipUtil.getProjectPaths() diff --git a/packages/core/src/testInteg/perf/prepareRepoData.test.ts b/packages/core/src/testInteg/perf/prepareRepoData.test.ts index 58b599f51a2..9485f132254 100644 --- a/packages/core/src/testInteg/perf/prepareRepoData.test.ts +++ b/packages/core/src/testInteg/perf/prepareRepoData.test.ts @@ -12,7 +12,7 @@ import { AmazonqCreateUpload, fs, getRandomString } from '../../shared' import { Span } from '../../shared/telemetry' import { FileSystem } from '../../shared/fs/fs' import AdmZip from 'adm-zip' -import { assertEfficientZip, getFsCallsUpperBound } from './utilities' +import { assertEfficientAdmZip, getFsCallsUpperBound } from './utilities' type resultType = { zipFileBuffer: Buffer @@ -89,7 +89,7 @@ function verifyResult(setup: setupResult, result: resultType, telemetry: Telemet assert.strictEqual(result.zipFileChecksum.length, 44) assert.ok(getFsCallsUpperBound(setup.fsSpy) <= setup.numFiles * 4, 'total system calls should be under 4 per file') - assertEfficientZip(setup.zipSpy, setup.numFiles) + assertEfficientAdmZip(setup.zipSpy, setup.numFiles) } describe('prepareRepoData', function () { diff --git a/packages/core/src/testInteg/perf/startSecurityScan.test.ts b/packages/core/src/testInteg/perf/startSecurityScan.test.ts index 79209322d31..ca68bccae6e 100644 --- a/packages/core/src/testInteg/perf/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/perf/startSecurityScan.test.ts @@ -21,9 +21,20 @@ import { } from '../../test/testUtil' import { getTestWindow } from '../../test/shared/vscode/window' import { SeverityLevel } from '../../test/shared/vscode/message' -import { CodeAnalysisScope } from '../../codewhisperer' +import { CodeAnalysisScope, ZipUtil } from '../../codewhisperer' import { performanceTest } from '../../shared/performance/performance' import { createClient } from '../../test/codewhisperer/testUtil' +import { fs } from '../../shared' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' + +interface SetupResult { + commandSpy: sinon.SinonSpy + securityScanRenderSpy: sinon.SinonSpy + zipUtil: ZipUtil + zipSpy: sinon.SinonSpiedInstance + fsSpy: sinon.SinonSpiedInstance +} describe('startSecurityScanPerformanceTest', function () { let extensionContext: FakeExtensionContext @@ -57,40 +68,65 @@ describe('startSecurityScanPerformanceTest', function () { }) } - performanceTest({}, 'Should calculate cpu and memory usage for file scans', function () { - return { - setup: async () => { - getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) - const commandSpy = sinon.spy(vscode.commands, 'executeCommand') - const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') - await model.CodeScansState.instance.setScansEnabled(true) - return { commandSpy, securityScanRenderSpy } + performanceTest( + { + darwin: { + userCpuUsage: 100, + systemCpuUsage: 35, + heapTotal: 2, }, - execute: async () => { - await startSecurityScan.startSecurityScan( - mockSecurityPanelViewProvider, - editor, - createClient(), - extensionContext, - CodeAnalysisScope.FILE - ) + linux: { + userCpuUsage: 100, + systemCpuUsage: 35, + heapTotal: 2, }, - verify: ({ - commandSpy, - securityScanRenderSpy, - }: { - commandSpy: sinon.SinonSpy - securityScanRenderSpy: sinon.SinonSpy - }) => { - assert.ok(commandSpy.neverCalledWith('workbench.action.problems.focus')) - assert.ok(securityScanRenderSpy.calledOnce) - const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) - assert.strictEqual(warnings.length, 0) - assertTelemetry('codewhisperer_securityScan', { - codewhispererCodeScanScope: 'FILE', - passive: true, - }) + win32: { + userCpuUsage: 100, + systemCpuUsage: 35, + heapTotal: 2, }, + }, + 'Should calculate cpu and memory usage for file scans', + function () { + return { + setup: async () => { + getFetchStubWithResponse({ status: 200, statusText: 'testing stub' }) + const commandSpy = sinon.spy(vscode.commands, 'executeCommand') + const securityScanRenderSpy = sinon.spy(diagnosticsProvider, 'initSecurityScanRender') + const zipUtil = new ZipUtil() + const zipSpy = sinon.spy(zipUtil) + const fsSpy = sinon.spy(fs) + await model.CodeScansState.instance.setScansEnabled(true) + return { commandSpy, securityScanRenderSpy, zipUtil, zipSpy, fsSpy } + }, + execute: async (setup: SetupResult) => { + await startSecurityScan.startSecurityScan( + mockSecurityPanelViewProvider, + editor, + createClient(), + extensionContext, + CodeAnalysisScope.FILE, + setup.zipUtil + ) + }, + verify: (setup: SetupResult) => { + assert.ok(setup.commandSpy.neverCalledWith('workbench.action.problems.focus')) + assert.ok(setup.securityScanRenderSpy.calledOnce) + + assert.ok(setup.zipSpy.generateZip.calledOnce) + assert.ok(setup.zipSpy.removeTmpFiles.calledOnce) + assert.ok( + getFsCallsUpperBound(setup.fsSpy) <= 5, + 'should make less than a small constant number of file system calls' + ) + const warnings = getTestWindow().shownMessages.filter((m) => m.severity === SeverityLevel.Warning) + assert.strictEqual(warnings.length, 0) + assertTelemetry('codewhisperer_securityScan', { + codewhispererCodeScanScope: 'FILE', + passive: true, + }) + }, + } } - }) + ) }) diff --git a/packages/core/src/testInteg/perf/utilities.ts b/packages/core/src/testInteg/perf/utilities.ts index 86ff69cc9c6..5c8199e5465 100644 --- a/packages/core/src/testInteg/perf/utilities.ts +++ b/packages/core/src/testInteg/perf/utilities.ts @@ -54,7 +54,7 @@ export function getFsWritesUpperBound(fsSpy: sinon.SinonSpiedInstance, numFiles: number): void | never { +export function assertEfficientAdmZip(zipSpy: sinon.SinonSpiedInstance, numFiles: number): void | never { assert.ok(zipSpy.addLocalFile.callCount <= numFiles, 'add files to zip at most once') assert.strictEqual(zipSpy.toBuffer.callCount, 1, 'creates buffer once') } From 10236d82fbf917176b7d951351d18570c9169a9e Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 15:50:15 -0400 Subject: [PATCH 21/27] add system spies for zipCode --- .../transformByQ/transformApiHandler.ts | 6 ++- packages/core/src/testInteg/perf/utilities.ts | 14 +++++-- .../core/src/testInteg/perf/zipcode.test.ts | 41 ++++++++++++------- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts index 4d8a3468d32..5556ae34f48 100644 --- a/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts +++ b/packages/core/src/codewhisperer/service/transformByQ/transformApiHandler.ts @@ -287,13 +287,15 @@ interface ZipCodeResult { fileSize: number } -export async function zipCode({ dependenciesFolder, humanInTheLoopFlag, modulePath, zipManifest }: IZipCodeParams) { +export async function zipCode( + { dependenciesFolder, humanInTheLoopFlag, modulePath, zipManifest }: IZipCodeParams, + zip: AdmZip = new AdmZip() +) { let tempFilePath = undefined let logFilePath = undefined let dependenciesCopied = false try { throwIfCancelled() - const zip = new AdmZip() // If no modulePath is passed in, we are not uploaded the source folder // NOTE: We only upload dependencies for human in the loop work diff --git a/packages/core/src/testInteg/perf/utilities.ts b/packages/core/src/testInteg/perf/utilities.ts index 5c8199e5465..01df7f3e2b2 100644 --- a/packages/core/src/testInteg/perf/utilities.ts +++ b/packages/core/src/testInteg/perf/utilities.ts @@ -50,11 +50,19 @@ export function getFsWritesUpperBound(fsSpy: sinon.SinonSpiedInstance, numFiles: number): void | never { - assert.ok(zipSpy.addLocalFile.callCount <= numFiles, 'add files to zip at most once') +export function assertEfficientAdmZip( + zipSpy: sinon.SinonSpiedInstance, + numFiles: number, + factor: number = 1, + offset: number = 0 +): void | never { + assert.ok( + zipSpy.addLocalFile.callCount <= factor * numFiles + offset, + 'add files to zip at factor*numFiles + offset times' + ) assert.strictEqual(zipSpy.toBuffer.callCount, 1, 'creates buffer once') } diff --git a/packages/core/src/testInteg/perf/zipcode.test.ts b/packages/core/src/testInteg/perf/zipcode.test.ts index 1883861f222..60f873c7085 100644 --- a/packages/core/src/testInteg/perf/zipcode.test.ts +++ b/packages/core/src/testInteg/perf/zipcode.test.ts @@ -10,12 +10,17 @@ import { createTestWorkspace } from '../../test/testUtil' import * as CodeWhispererConstants from '../../codewhisperer/models/constants' import { performanceTest } from '../../shared/performance/performance' import { zipCode } from '../../codewhisperer/indexNode' +import { FileSystem } from '../../shared/fs/fs' +import AdmZip from 'adm-zip' +import { assertEfficientAdmZip, getFsCallsUpperBound } from './utilities' interface SetupResult { tempDir: string tempFileName: string transformQManifest: ZipManifest - writeSpy: sinon.SinonSpy + fsSpy: sinon.SinonSpiedInstance + zipSpy: sinon.SinonSpiedInstance + zip: AdmZip } async function setup(numberOfFiles: number, fileSize: number): Promise { @@ -28,11 +33,13 @@ async function setup(numberOfFiles: number, fileSize: number): Promise await setup(numberOfFiles, fileSize), - execute: async ({ tempDir, tempFileName, transformQManifest, writeSpy }: SetupResult) => { - await zipCode({ - dependenciesFolder: { - path: tempDir, - name: tempFileName, + execute: async (setup: SetupResult) => { + await zipCode( + { + dependenciesFolder: { + path: setup.tempDir, + name: setup.tempFileName, + }, + humanInTheLoopFlag: false, + modulePath: setup.tempDir, + zipManifest: setup.transformQManifest, }, - humanInTheLoopFlag: false, - modulePath: tempDir, - zipManifest: transformQManifest, - }) + setup.zip + ) }, verify: async (setup: SetupResult) => { assert.ok( - setup.writeSpy.args.find((arg) => { - return arg[0].endsWith('.zip') + setup.fsSpy.writeFile.args.find((arg) => { + return arg[0].toString().endsWith('.zip') }) ) + + assert.ok(getFsCallsUpperBound(setup.fsSpy) <= 15) + assertEfficientAdmZip(setup.zipSpy, numberOfFiles, 2, 1) }, } } From efe25af26cd00ed4f1bcecdeee91d808e8335e39 Mon Sep 17 00:00:00 2001 From: hkobew Date: Mon, 14 Oct 2024 17:07:09 -0400 Subject: [PATCH 22/27] add filesystem spy --- .../src/testInteg/perf/tryInstallLsp.test.ts | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts index 7cc9c18b2c4..30957b65d0b 100644 --- a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts +++ b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts @@ -11,6 +11,8 @@ import { LspController } from '../../amazonq' import { fs, getRandomString, globals } from '../../shared' import { createTestWorkspace } from '../../test/testUtil' import { performanceTest } from '../../shared/performance/performance' +import { getFsCallsUpperBound } from './utilities' +import { FileSystem } from '../../shared/fs/fs' // fakeFileContent is matched to fakeQServerContent based on hash. const fakeHash = '4eb2865c8f40a322aa04e17d8d83bdaa605d6f1cb363af615240a5442a010e0aef66e21bcf4c88f20fabff06efe8a214' @@ -31,7 +33,7 @@ const fakeNodeContent = { serverVersion: '1.1.1', } -function createStubs(numberOfFiles: number, fileSize: number) { +function createStubs(numberOfFiles: number, fileSize: number): sinon.SinonSpiedInstance { // Avoid making HTTP request or mocking giant manifest, stub what we need directly from request. sinon.stub(LspController.prototype, 'fetchManifest') // Directly feed the runtime specifications. @@ -41,8 +43,11 @@ function createStubs(numberOfFiles: number, fileSize: number) { sinon.stub(LspController.prototype, '_download').callsFake(getFakeDownload(numberOfFiles, fileSize)) // Hard code the hash since we are creating files on the spot, whose hashes can't be predicted. sinon.stub(LspController.prototype, 'getFileSha384').resolves(fakeHash) - // Don't allow tryInstallLsp to move runtimes out of temporary folder. - sinon.stub(fs, 'rename') + const fsSpy = sinon.spy(fs) + fsSpy.rename.restore() + // Don't allow tryInstallLsp to move runtimes out of temporary folder + sinon.stub(fsSpy, 'rename') + return fsSpy } /** @@ -66,7 +71,7 @@ const getFakeDownload = function (numberOfFiles: number, fileSize: number) { } } -function performanceTestWrapper(numFiles: number, fileSize: number) { +function performanceTestWrapper(numFiles: number, fileSize: number, message: string) { return performanceTest( { testRuns: 10, @@ -89,17 +94,18 @@ function performanceTestWrapper(numFiles: number, fileSize: number) { duration: 15, }, }, - 'many small files in zip', + message, function () { return { setup: async () => { - createStubs(numFiles, fileSize) + return createStubs(numFiles, fileSize) }, execute: async () => { return await LspController.instance.tryInstallLsp(globals.context) }, - verify: async (_setup: any, result: boolean) => { + verify: async (fsSpy: sinon.SinonSpiedInstance, result: boolean) => { assert.ok(result) + assert.ok(getFsCallsUpperBound(fsSpy) <= 6 * numFiles) }, } } @@ -111,7 +117,7 @@ describe('tryInstallLsp', function () { sinon.restore() }) describe('performance tests', function () { - performanceTestWrapper(250, 10) - performanceTestWrapper(10, 1000) + performanceTestWrapper(250, 10, '250x10') + performanceTestWrapper(10, 1000, '10x1000') }) }) From 3a11a8576eab90ebb3743475b5178e9e590e8658 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 15 Oct 2024 13:18:14 -0400 Subject: [PATCH 23/27] add spy to file hash test --- .../src/testInteg/perf/getFileSha384.test.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/core/src/testInteg/perf/getFileSha384.test.ts b/packages/core/src/testInteg/perf/getFileSha384.test.ts index 5072f448244..b2afa1e3975 100644 --- a/packages/core/src/testInteg/perf/getFileSha384.test.ts +++ b/packages/core/src/testInteg/perf/getFileSha384.test.ts @@ -4,10 +4,18 @@ */ import assert from 'assert' import path from 'path' +import sinon from 'sinon' import { getTestWorkspaceFolder } from '../integrationTestsUtilities' import { fs, getRandomString } from '../../shared' import { LspController } from '../../amazonq' import { performanceTest } from '../../shared/performance/performance' +import { FileSystem } from '../../shared/fs/fs' +import { getFsCallsUpperBound } from './utilities' + +interface SetupResult { + testFile: string + fsSpy: sinon.SinonSpiedInstance +} function performanceTestWrapper(label: string, fileSize: number) { return performanceTest( @@ -37,14 +45,15 @@ function performanceTestWrapper(label: string, fileSize: number) { const fileContent = getRandomString(fileSize) const testFile = path.join(workspace, 'test-file') await fs.writeFile(testFile, fileContent) - - return testFile + const fsSpy = sinon.spy(fs) + return { testFile, fsSpy } }, - execute: async (testFile: string) => { - return await LspController.instance.getFileSha384(testFile) + execute: async (setup: SetupResult) => { + return await LspController.instance.getFileSha384(setup.testFile) }, - verify: async (_testFile: string, result: string) => { + verify: async (setup: SetupResult, result: string) => { assert.strictEqual(result.length, 96) + assert.ok(getFsCallsUpperBound(setup.fsSpy) <= 1, 'makes a single call to fs') }, } } @@ -53,6 +62,9 @@ function performanceTestWrapper(label: string, fileSize: number) { describe('getFileSha384', function () { describe('performance tests', function () { + afterEach(function () { + sinon.restore() + }) performanceTestWrapper('1MB', 1000) performanceTestWrapper('2MB', 2000) performanceTestWrapper('4MB', 4000) From 1a2017b973b4cb64e575654ac45c30d8ba88c767 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 15 Oct 2024 13:27:37 -0400 Subject: [PATCH 24/27] add spy for vfs --- .../testInteg/perf/registerNewFiles.test.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/core/src/testInteg/perf/registerNewFiles.test.ts b/packages/core/src/testInteg/perf/registerNewFiles.test.ts index d658bf39ded..9c8415f3164 100644 --- a/packages/core/src/testInteg/perf/registerNewFiles.test.ts +++ b/packages/core/src/testInteg/perf/registerNewFiles.test.ts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ import assert from 'assert' +import sinon from 'sinon' import * as vscode from 'vscode' import { NewFileInfo, NewFileZipContents, registerNewFiles } from '../../amazonqFeatureDev' import { performanceTest } from '../../shared/performance/performance' @@ -12,6 +13,8 @@ import { VirtualFileSystem } from '../../shared' interface SetupResult { workspace: vscode.WorkspaceFolder fileContents: NewFileZipContents[] + vfsSpy: sinon.SinonSpiedInstance + vfs: VirtualFileSystem } function getFileContents(numFiles: number, fileSize: number): NewFileZipContents[] { @@ -31,17 +34,17 @@ function performanceTestWrapper(label: string, numFiles: number, fileSize: numbe linux: { userCpuUsage: 200, systemCpuUsage: 35, - heapTotal: 8, + heapTotal: 16, }, darwin: { userCpuUsage: 200, systemCpuUsage: 35, - heapTotal: 8, + heapTotal: 16, }, win32: { userCpuUsage: 200, systemCpuUsage: 35, - heapTotal: 8, + heapTotal: 16, }, }, label, @@ -50,6 +53,9 @@ function performanceTestWrapper(label: string, numFiles: number, fileSize: numbe setup: async () => { const testWorkspaceUri = vscode.Uri.file(getTestWorkspaceFolder()) const fileContents = getFileContents(numFiles, fileSize) + const vfs = new VirtualFileSystem() + const vfsSpy = sinon.spy(vfs) + return { workspace: { uri: testWorkspaceUri, @@ -57,19 +63,25 @@ function performanceTestWrapper(label: string, numFiles: number, fileSize: numbe index: 0, }, fileContents: fileContents, + vfsSpy: vfsSpy, + vfs: vfs, } }, execute: async (setup: SetupResult) => { return registerNewFiles( - new VirtualFileSystem(), + setup.vfs, setup.fileContents, 'test-upload-id', [setup.workspace], conversationId ) }, - verify: async (_setup: SetupResult, result: NewFileInfo[]) => { + verify: async (setup: SetupResult, result: NewFileInfo[]) => { assert.strictEqual(result.length, numFiles) + assert.ok( + setup.vfsSpy.registerProvider.callCount <= numFiles, + 'only register each file once in vfs' + ) }, } } From 5e2d976f113fdb78157afd04d0f24c9dca21bf44 Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 15 Oct 2024 17:41:03 -0400 Subject: [PATCH 25/27] adjust thresholds --- packages/core/src/testInteg/perf/startSecurityScan.test.ts | 6 +++--- packages/core/src/testInteg/perf/tryInstallLsp.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/src/testInteg/perf/startSecurityScan.test.ts b/packages/core/src/testInteg/perf/startSecurityScan.test.ts index ca68bccae6e..2b1da3d8452 100644 --- a/packages/core/src/testInteg/perf/startSecurityScan.test.ts +++ b/packages/core/src/testInteg/perf/startSecurityScan.test.ts @@ -71,17 +71,17 @@ describe('startSecurityScanPerformanceTest', function () { performanceTest( { darwin: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 2, }, linux: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 2, }, win32: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 2, }, diff --git a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts index 30957b65d0b..5c96840a2a2 100644 --- a/packages/core/src/testInteg/perf/tryInstallLsp.test.ts +++ b/packages/core/src/testInteg/perf/tryInstallLsp.test.ts @@ -76,19 +76,19 @@ function performanceTestWrapper(numFiles: number, fileSize: number, message: str { testRuns: 10, linux: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 6, duration: 15, }, darwin: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 6, duration: 15, }, win32: { - userCpuUsage: 100, + userCpuUsage: 150, systemCpuUsage: 35, heapTotal: 6, duration: 15, From 157c628811df41d0745457e9c1d7f70edc119f0b Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 15 Oct 2024 17:55:10 -0400 Subject: [PATCH 26/27] remove temp debump to 1 testrun --- packages/core/src/testInteg/perf/getFileSha384.test.ts | 2 +- packages/core/src/testInteg/perf/prepareRepoData.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/testInteg/perf/getFileSha384.test.ts b/packages/core/src/testInteg/perf/getFileSha384.test.ts index b2afa1e3975..ea9774edbf2 100644 --- a/packages/core/src/testInteg/perf/getFileSha384.test.ts +++ b/packages/core/src/testInteg/perf/getFileSha384.test.ts @@ -20,7 +20,7 @@ interface SetupResult { function performanceTestWrapper(label: string, fileSize: number) { return performanceTest( { - testRuns: 1, + testRuns: 10, linux: { userCpuUsage: 400, systemCpuUsage: 35, diff --git a/packages/core/src/testInteg/perf/prepareRepoData.test.ts b/packages/core/src/testInteg/perf/prepareRepoData.test.ts index 9485f132254..589b044d245 100644 --- a/packages/core/src/testInteg/perf/prepareRepoData.test.ts +++ b/packages/core/src/testInteg/perf/prepareRepoData.test.ts @@ -31,7 +31,7 @@ type setupResult = { function performanceTestWrapper(numFiles: number, fileSize: number) { return performanceTest( { - testRuns: 1, + testRuns: 10, linux: { userCpuUsage: 150, systemCpuUsage: 35, From 6ecc7081a56739aa661703fcbcf440a71fd4929e Mon Sep 17 00:00:00 2001 From: hkobew Date: Tue, 15 Oct 2024 18:38:24 -0400 Subject: [PATCH 27/27] adjust thresholds --- packages/core/src/testInteg/perf/registerNewFiles.test.ts | 6 +++--- packages/core/src/testInteg/perf/zipcode.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/src/testInteg/perf/registerNewFiles.test.ts b/packages/core/src/testInteg/perf/registerNewFiles.test.ts index 9c8415f3164..a0133de382e 100644 --- a/packages/core/src/testInteg/perf/registerNewFiles.test.ts +++ b/packages/core/src/testInteg/perf/registerNewFiles.test.ts @@ -34,17 +34,17 @@ function performanceTestWrapper(label: string, numFiles: number, fileSize: numbe linux: { userCpuUsage: 200, systemCpuUsage: 35, - heapTotal: 16, + heapTotal: 20, }, darwin: { userCpuUsage: 200, systemCpuUsage: 35, - heapTotal: 16, + heapTotal: 20, }, win32: { userCpuUsage: 200, systemCpuUsage: 35, - heapTotal: 16, + heapTotal: 20, }, }, label, diff --git a/packages/core/src/testInteg/perf/zipcode.test.ts b/packages/core/src/testInteg/perf/zipcode.test.ts index 60f873c7085..21a172e6c9f 100644 --- a/packages/core/src/testInteg/perf/zipcode.test.ts +++ b/packages/core/src/testInteg/perf/zipcode.test.ts @@ -47,17 +47,17 @@ function performanceTestWrapper(numberOfFiles: number, fileSize: number) { { testRuns: 10, linux: { - userCpuUsage: 120, + userCpuUsage: 200, systemCpuUsage: 50, heapTotal: 4, }, darwin: { - userCpuUsage: 120, + userCpuUsage: 200, systemCpuUsage: 50, heapTotal: 4, }, win32: { - userCpuUsage: 120, + userCpuUsage: 200, systemCpuUsage: 50, heapTotal: 4, },