[port 2.0]: Fix upload release report step (#22167)

Cherry-pick:

#21887
#21969
#22125

[Test
Run](https://dev.azure.com/fluidframework/internal/_build/results?buildId=286067&view=results)

build-tools/packages/build-cli/src/commands/release/report-unreleased.ts

@@ -5,7 +5,7 @@
 
 import * as fs from "node:fs/promises";
 import path from "node:path";
-import { isInternalVersionRange, isInternalVersionScheme } from "@fluid-tools/version-tools";
+import { isInternalTestVersion } from "@fluid-tools/version-tools";
 import type { Logger } from "@fluidframework/build-tools";
 import { Flags } from "@oclif/core";
 import { formatISO } from "date-fns";
@@ -154,19 +154,41 @@ async function updateReportVersions(
 	version: string,
 	log: Logger,
 ): Promise<void> {
+	const clientPackageName = "fluid-framework";
+
+	const packageReleaseDetails = report[clientPackageName];
+
+	if (packageReleaseDetails === undefined) {
+		throw new Error(`Client package ${clientPackageName} is not defined in the report.`);
+	}
+
+	if (packageReleaseDetails.ranges?.caret === undefined) {
+		throw new Error(`Caret version for ${clientPackageName} is not defined in the report.`);
+	}
+
+	if (packageReleaseDetails.version === undefined) {
+		throw new Error(`Simple version for ${clientPackageName} is not defined in the report.`);
+	}
+
+	const clientVersionCaret = report[clientPackageName].ranges.caret;
+	const clientVersionSimple = report[clientPackageName].version;
+
+	log.log(`Caret version: ${clientVersionCaret}`);
+	log.log(`Simple version: ${clientVersionSimple}`);
+
 	for (const packageName of Object.keys(report)) {
 		if (ignorePackageList.has(packageName)) {
 			continue;
 		}
 
-		// updates caret ranges
-		if (isInternalVersionRange(report[packageName].ranges.caret, true)) {
-			// If the caret range is a range, reset it to an exact version.
-			// Note: Post 2.0 release, the versions will no longer be internal versions so another condition will be required that will work after 2.0.
+		const packageInfo = report[packageName];
+
+		// todo: add better checks
+		if (packageInfo.ranges.caret && packageInfo.ranges.caret === clientVersionCaret) {
 			report[packageName].ranges.caret = version;
 		}
 
-		if (isInternalVersionScheme(report[packageName].version)) {
+		if (packageInfo.version && packageInfo.version === clientVersionSimple) {
 			report[packageName].version = version;
 		}
 	}
@@ -185,7 +207,12 @@ async function updateReportVersions(
  */
 
 function extractBuildNumber(version: string): number {
-	const versionParts: string[] = version.split(".");
+	const versionParts: string[] = version.split("-");
+
+	if (isInternalTestVersion(version)) {
+		return Number.parseInt(versionParts[1], 10);
+	}
+
 	// Extract the last part of the version, which is the number you're looking for
 	return Number.parseInt(versionParts[versionParts.length - 1], 10);
 }

build-tools/packages/version-tools/api-report/version-tools.api.md

@@ -60,6 +60,9 @@ export type InterdependencyRange = WorkspaceRange | RangeOperator | RangeOperato
 // @public
 export function isInterdependencyRange(r: unknown): r is InterdependencyRange;
 
+// @public
+export function isInternalTestVersion(version: semver.SemVer | string): boolean;
+
 // @public
 export function isInternalVersionRange(range: string, allowAnyPrereleaseId?: boolean): boolean;
 

build-tools/packages/version-tools/src/index.ts

@@ -27,6 +27,7 @@ export {
 	changePreReleaseIdentifier,
 	getVersionRange,
 	fromInternalScheme,
+	isInternalTestVersion,
 	isInternalVersionRange,
 	isInternalVersionScheme,
 	toInternalScheme,

build-tools/packages/version-tools/src/internalVersionScheme.ts

@@ -490,3 +490,44 @@ export function detectInternalVersionConstraintType(
 	const maxSatisfying = semver.maxSatisfying([patch, minor], range);
 	return maxSatisfying === patch ? "patch" : maxSatisfying === minor ? "minor" : "exact";
 }
+
+/**
+ * Checks if the provided version is a test version.
+ *
+ * Test versions are generated from test/ branches and are published to the test feed.
+ *
+ * @param version - The version to check
+ * @returns - True if the version string is a test version, otherwise false
+ *
+ * @example
+ * returns true
+ * isInternalTestVersion("0.0.0-260312-test");
+ *
+ * @example
+ * returns false
+ * isInternalTestVersion("2.1.0-260312");
+ *
+ * @throws error - If the version string cannot be parsed as a valid semantic version.
+ */
+export function isInternalTestVersion(version: semver.SemVer | string): boolean {
+	const parsedVersion = semver.parse(version);
+
+	if (parsedVersion === null) {
+		throw new Error(`Couldn't parse ${version} as a semver.`);
+	}
+
+	if (
+		parsedVersion.prerelease.length === 0 ||
+		typeof parsedVersion.prerelease[0] !== "string"
+	) {
+		return false;
+	}
+
+	const isTestVersion =
+		parsedVersion.minor === 0 &&
+		parsedVersion.major === 0 &&
+		parsedVersion.patch === 0 &&
+		parsedVersion.prerelease[0].endsWith("-test");
+
+	return isTestVersion;
+}

build-tools/packages/version-tools/src/test/internalVersionScheme.test.ts

@@ -10,6 +10,7 @@ import {
 	detectInternalVersionConstraintType,
 	fromInternalScheme,
 	getVersionRange,
+	isInternalTestVersion,
 	isInternalVersionRange,
 	isInternalVersionScheme,
 	toInternalScheme,
@@ -387,4 +388,60 @@ describe("internalScheme", () => {
 			);
 		});
 	});
+
+	describe("checking test version scheme", () => {
+		it("0.0.0-285010-test is a valid test version", () => {
+			const input = `0.0.0-285010-test`;
+			const result = isInternalTestVersion(input);
+			assert.isTrue(result);
+		});
+
+		it("0.0.0-test-285010 is not a valid test version", () => {
+			const input = `0.0.0-test-285010`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+
+		it("2.1.0-test-285010 is not a valid test version", () => {
+			const input = `2.1.0-test-285010`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+
+		it("2.1.0-285010-test is not a valid test version", () => {
+			const input = `2.1.0-285010-test`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+
+		it("2.1.0-285010 is a prerelease version but not a test version", () => {
+			const input = `2.1.0-285010`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+
+		it("2.1.0 is a release version but not a test version", () => {
+			const input = `2.1.0`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+
+		it("2.0.0-internal.1.0.0 is internal scheme but not a test version", () => {
+			const input = `2.0.0-internal.1.0.0`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+
+		it("2.0.0-rc.1.0.0 is internal scheme but not a test version", () => {
+			const input = `2.0.0-rc.1.0.0`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+
+		it("2.0.0-dev.1.1.0.123 is an internal dev version but not a test version", () => {
+			const input = `2.0.0-dev.1.1.0.123`;
+			const result = isInternalTestVersion(input);
+			assert.isFalse(result);
+		});
+	});
 });

tools/pipelines/server-gitrest.yml

@@ -80,7 +80,7 @@ pr:
     - tools/pipelines/templates/include-vars.yml
     - tools/pipelines/templates/include-install-pnpm.yml
     - tools/pipelines/templates/include-use-node-version.yml
-    - tools/pipelines/templates/upload-dev-manifest.yml
+
 
 variables:
   - template: /tools/pipelines/templates/include-vars.yml@self

tools/pipelines/server-historian.yml

@@ -80,7 +80,6 @@ pr:
     - tools/pipelines/templates/include-vars.yml
     - tools/pipelines/templates/include-install-pnpm.yml
     - tools/pipelines/templates/include-use-node-version.yml
-    - tools/pipelines/templates/upload-dev-manifest.yml
 
 variables:
   - template: /tools/pipelines/templates/include-vars.yml@self

tools/pipelines/server-routerlicious.yml

@@ -94,7 +94,6 @@ pr:
     - tools/pipelines/templates/include-vars.yml
     - tools/pipelines/templates/include-install-pnpm.yml
     - tools/pipelines/templates/include-use-node-version.yml
-    - tools/pipelines/templates/upload-dev-manifest.yml
     exclude:
     - server/routerlicious/kubernetes/routerlicious
 

tools/pipelines/templates/build-docker-service.yml

@@ -548,6 +548,7 @@ extends:
         parameters:
           tagName: ${{ parameters.tagName }}
           isReleaseGroup: ${{ parameters.isReleaseGroup }}
+          buildDirectory: ${{ parameters.buildDirectory }}
 
     - ${{ if eq(parameters.shouldReleaseDockerImage, true) }}:
       - template: /tools/pipelines/templates/include-publish-docker-service.yml@self

tools/pipelines/templates/build-npm-package.yml

@@ -334,8 +334,6 @@ extends:
                       tagName: '${{ parameters.tagName }}'
                       interdependencyRange: '${{ parameters.interdependencyRange }}'
                       packageTypesOverride: '${{ parameters.packageTypesOverride }}'
-                      STORAGE_ACCOUNT: $(STORAGE_ACCOUNT)
-                      STORAGE_KEY: $(STORAGE_KEY)
 
               # Build and Lint
               - template: /tools/pipelines/templates/include-build-lint.yml@self
@@ -582,16 +580,6 @@ extends:
                       sbomEnabled: false
                       publishLocation: pipeline
 
-                # This condition should be kept in sync with the one in include-set-package-version, which decides if
-                # upload-dev-manifest.yml should be included or not.
-                - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.tagName, 'client'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'))) }}:
-                  - output: pipelineArtifact
-                    displayName: Publish Artifact - release_reports
-                    targetPath: $(Build.ArtifactStagingDirectory)/release_reports
-                    artifactName: release_reports
-                    sbomEnabled: false
-                    publishLocation: pipeline
-
           # Job - Component detection
           - ${{ if eq(parameters.publish, true) }}:
               - job: CG
@@ -617,6 +605,7 @@ extends:
           parameters:
             tagName: ${{ parameters.tagName }}
             isReleaseGroup: ${{ parameters.isReleaseGroup }}
+            buildDirectory: ${{ parameters.buildDirectory }}
 
       # Capture pipeline stage results
       - ${{ if eq(parameters.telemetry, true) }}:

tools/pipelines/templates/include-install-build-tools.yml

@@ -0,0 +1,70 @@
+# Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+# Licensed under the MIT License.
+
+# include-install-build-tools
+#
+# This template can be included in pipelines to install the Fluid build-tools locally so flub can be used in the
+# pipeline.
+
+parameters:
+- name: buildDirectory
+  type: string
+
+- name: buildToolsVersionToInstall
+  type: string
+  default: repo
+
+# The path to the pnpm store.
+- name: pnpmStorePath
+  type: string
+  default: $(Pipeline.Workspace)/.pnpm-store
+
+steps:
+
+  # These steps should ONLY run if we're using the repo version of the build tools. These steps are mutually exclusive
+  # with the next group of steps.
+  - ${{ if eq(parameters.buildToolsVersionToInstall, 'repo') }}:
+    - template: include-install-pnpm.yml
+      parameters:
+        buildDirectory: $(Build.SourcesDirectory)/build-tools
+        pnpmStorePath: ${{ parameters.pnpmStorePath }}
+        enableCache: false
+
+    - task: Bash@3
+      name: InstallBuildTools
+      displayName: Install Fluid Build Tools (from repo)
+      inputs:
+        targetType: 'inline'
+        workingDirectory: $(Build.SourcesDirectory)/build-tools
+        script: |
+          pnpm i --frozen-lockfile
+          pnpm build:compile
+          cd packages/build-cli
+          # Use npm link instead of pnpm link because it handles bins better
+          npm link
+          echo "which flub: $(which flub)"
+
+  # These steps install a version of build-tools from the npm registry. As noted above, these steps are mutually exclusive
+  # with the previous group of steps.
+  - ${{ if ne(parameters.buildToolsVersionToInstall, 'repo') }}:
+    - task: Bash@3
+      name: InstallBuildTools
+      displayName: Install Fluid Build Tools (from npm)
+      inputs:
+        targetType: 'inline'
+        workingDirectory: ${{ parameters.buildDirectory }}
+        script: |
+          echo "${{ parameters.buildToolsVersionToInstall }}"
+          npm install --global "@fluid-tools/build-cli@${{ parameters.buildToolsVersionToInstall }}"
+
+  - task: Bash@3
+    name: BuildToolsInstallCheck
+    displayName: Check Build Tools Installation
+    inputs:
+      targetType: 'inline'
+      workingDirectory: ${{ parameters.buildDirectory }}
+      script: |
+        # Output the help and full command list for debugging purposes
+        echo "which flub: $(which flub)"
+        flub --help
+        flub commands

tools/pipelines/templates/include-publish-npm-package.yml

@@ -15,6 +15,9 @@ parameters:
   type: boolean
   default: false
 
+- name: buildDirectory
+  type: string
+
 stages:
 
 # This stage only runs for test-branch builds
@@ -32,6 +35,14 @@ stages:
       environment: test-package-build-feed
       pool: ${{ parameters.pool }}
 
+  # Only generate manifest files for runs in the internal project (i.e. CI runs, not PR run), for the main, test and release branch, and
+  # for the build of the client release group (we don't need manifests for anything else).
+  # Enabling this template for every PR run risks overwriting existing manifest files uploaded to Azure blobs. Therefore, it's crucial to restrict this template to commits merged into the main and release branch.
+  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.tagName, 'client'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/test/'))) }}:
+    - template: /tools/pipelines/templates/upload-dev-manifest.yml@self
+      parameters:
+        buildDirectory: '${{ parameters.buildDirectory }}'
+
 # This stage and the following stage both run for non-test builds. The build feed is equivalent to the public npmjs.org
 # feed except it contains builds for every main- or next-branch build.
 - stage: publish_npm_internal_build
@@ -50,6 +61,14 @@ stages:
       environment: package-build-feed
       pool: ${{ parameters.pool }}
 
+  # Only generate manifest files for runs in the internal project (i.e. CI runs, not PR run), for the main, test and release branch, and
+  # for the build of the client release group (we don't need manifests for anything else).
+  # Enabling this template for every PR run risks overwriting existing manifest files uploaded to Azure blobs. Therefore, it's crucial to restrict this template to commits merged into the main and release branch.
+  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.tagName, 'client'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/test/'))) }}:
+    - template: /tools/pipelines/templates/upload-dev-manifest.yml@self
+      parameters:
+        buildDirectory: '${{ parameters.buildDirectory }}'
+
 # Only add a stage to publish packages to the dev feed for release groups, which have separate lists of packages to
 # publish to each feed. Non-release-group builds (for an independent package) only need to publish to the build feed.
 - ${{ if eq(parameters.isReleaseGroup, true) }}: