[ci][microcheck] add macos + windows tests to microcheck (ray-project…

…#45361)

Add macos + window tests to microcheck

Test:
- CI

---------

Signed-off-by: can <[email protected]>

.buildkite/others.rayci.yml

@@ -67,7 +67,10 @@ steps:
     if: build.branch == "master"
     instance_type: small
     commands:
-      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --production
+      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix linux:__ --production
+      # we use master branch to determine microcheck tests for darwin and windows, because darwin and windows tests do not run on premerge branches
+      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix darwin:__ --consider-master-branch --production
+      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix windows:__ --consider-master-branch --production
     matrix:
       - "core"
       - "serverless"

ci/ray_ci/automation/determine_microcheck_tests.py

@@ -17,12 +17,14 @@
 @click.option("--test-history-length", default=500, type=int)
 @click.option("--test-prefix", default=LINUX_TEST_PREFIX, type=str)
 @click.option("--production", is_flag=True, default=False)
+@click.option("--consider-master-branch", is_flag=True, default=False)
 def main(
     team: str,
     coverage: int,
     test_history_length: int,
     test_prefix: str,
     production: bool,
+    consider_master_branch: bool,
 ) -> None:
     """
     This script determines the tests that need to be run to cover a certain percentage
@@ -40,6 +42,10 @@ def main(
         test.get_name(): _get_failed_prs(test, test_history_length) for test in tests
     }
     high_impact_tests = _get_test_with_minimal_coverage(test_to_prs, coverage)
+    if consider_master_branch:
+        high_impact_tests = high_impact_tests.union(
+            _get_failed_tests_from_master_branch(tests, test_history_length)
+        )
     if production:
         _update_high_impact_tests(tests, high_impact_tests)
 
@@ -60,6 +66,38 @@ def _update_high_impact_tests(tests: List[Test], high_impact_tests: Set[str]) ->
         test.persist_to_s3()
 
 
+def _get_failed_tests_from_master_branch(
+    tests: List[Test], test_history_length: int
+) -> Set[str]:
+    """
+    Get the tests that failed on the master branch
+    """
+    failed_tests = set()
+    for test in tests:
+        results = [
+            result
+            for result in test.get_test_results(
+                limit=test_history_length,
+                aws_bucket=get_global_config()["state_machine_branch_aws_bucket"],
+                use_async=True,
+                refresh=True,
+            )
+            if result.branch == "master"
+        ]
+        consecutive_failures = 0
+        # If a test fails 2 times in a row, we consider it as a failed test
+        for result in results:
+            if result.status == ResultStatus.ERROR.value:
+                consecutive_failures += 1
+            else:
+                consecutive_failures = 0
+            if consecutive_failures == 2:
+                failed_tests.add(test.get_name())
+                break
+
+    return failed_tests
+
+
 def _get_test_with_minimal_coverage(
     test_to_prs: Dict[str, Set[str]], coverage: int
 ) -> Set[str]:
@@ -117,6 +155,7 @@ def _get_failed_prs(test: Test, test_history_length: int) -> Set[str]:
             limit=test_history_length,
             aws_bucket=get_global_config()["state_machine_pr_aws_bucket"],
             use_async=True,
+            refresh=True,
         )
         if result.status == ResultStatus.ERROR.value
     ]

ci/ray_ci/automation/test_determine_microcheck_tests.py

@@ -7,6 +7,7 @@
 from ci.ray_ci.automation.determine_microcheck_tests import (
     _get_failed_prs,
     _get_test_with_minimal_coverage,
+    _get_failed_tests_from_master_branch,
     _update_high_impact_tests,
 )
 from ci.ray_ci.utils import ci_init
@@ -26,7 +27,7 @@ def get_name(self) -> str:
         return self.get("name", "")
 
     def get_test_results(
-        self, limit: int, aws_bucket: str, use_async: bool
+        self, limit: int, aws_bucket: str, use_async: bool, refresh: bool
     ) -> List[TestResult]:
         return self.get("test_results", [])
 
@@ -83,6 +84,44 @@ def test_get_failed_prs():
     ) == {"w00t", "f00"}
 
 
+def test_get_failed_tests_from_master_branch():
+    failed_test_01 = MockTest(
+        {
+            "name": "test_01",
+            "test_results": [
+                stub_test_result(ResultStatus.ERROR, "master"),
+                stub_test_result(ResultStatus.SUCCESS, "master"),
+                stub_test_result(ResultStatus.ERROR, "master"),
+                stub_test_result(ResultStatus.ERROR, "master"),
+            ],
+        },
+    )
+    failed_test_02 = MockTest(
+        {
+            "name": "test_02",
+            "test_results": [
+                stub_test_result(ResultStatus.ERROR, "non_master"),
+                stub_test_result(ResultStatus.SUCCESS, "non_master"),
+                stub_test_result(ResultStatus.ERROR, "non_master"),
+                stub_test_result(ResultStatus.ERROR, "non_master"),
+            ],
+        },
+    )
+    failed_test_03 = MockTest(
+        {
+            "name": "test_03",
+            "test_results": [
+                stub_test_result(ResultStatus.ERROR, "master"),
+                stub_test_result(ResultStatus.SUCCESS, "master"),
+                stub_test_result(ResultStatus.ERROR, "master"),
+            ],
+        },
+    )
+    _get_failed_tests_from_master_branch(
+        [failed_test_01, failed_test_02, failed_test_03], 2
+    ) == {"test_01"}
+
+
 def test_get_test_with_minimal_coverage():
     # empty cases
     assert _get_test_with_minimal_coverage({}, 50) == set()