1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <fcntl.h>
18 #if !defined(__APPLE__)
19 #include <malloc.h>
20 #endif
21 #include <signal.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <sys/types.h>
25 #include <sys/wait.h>
26 #include <unistd.h>
27 
28 #include <map>
29 #include <regex>
30 #include <sstream>
31 #include <string>
32 #include <vector>
33 
34 #include <android-base/file.h>
35 #include <android-base/strings.h>
36 #include <android-base/test_utils.h>
37 #include <gtest/gtest.h>
38 
39 #include "NanoTime.h"
40 
41 #if defined(__APPLE__)
42 extern char** environ;
43 
clearenv()44 int clearenv() {
45   *environ = nullptr;
46   return 0;
47 }
48 #endif
49 
50 // Change the slow threshold for these tests since a few can take around
51 // 20 seconds.
GetInitialArgs(const char *** args,size_t * num_args)52 extern "C" bool GetInitialArgs(const char*** args, size_t* num_args) {
53   static const char* initial_args[] = {"--slow_threshold_ms=25000"};
54   *args = initial_args;
55   *num_args = 1;
56   return true;
57 }
58 
59 namespace android {
60 namespace gtest_extras {
61 
62 class SystemTests : public ::testing::Test {
63  protected:
SetUp()64   void SetUp() override {
65     raw_output_ = "";
66     sanitized_output_ = "";
67     exitcode_ = 0;
68 
69     // Clear all environment variables to make sure the test isn't affected
70     // by GTEST_XXX ones.
71     clearenv();
72   }
73 
74   void SanitizeOutput();
75 
76   void Exec(std::vector<const char*> args);
77   void ExecAndCapture(std::vector<const char*> args);
78   void RunTest(const std::string& test_name, std::vector<const char*> extra_args = {});
79   void RunTestCaptureFooter(const std::string& test_name, std::string* footer,
80                             std::vector<const char*> extra_args = {});
81   void Verify(const std::string& test_name, const std::string& expected_output,
82               int expected_exitcode, std::vector<const char*> extra_args = {});
83   void VerifySortedOutput(const std::string& test_name, const std::string& expected_output,
84                           int expected_exitcode, std::vector<const char*> extra_args = {});
85 
86   std::string raw_output_;
87   std::string sanitized_output_;
88   int exitcode_;
89   pid_t pid_;
90   int fd_;
91 };
92 
SortTestOutput(const std::string & output)93 static std::string SortTestOutput(const std::string& output) {
94   std::map<std::string, std::string> tests;
95 
96   std::vector<std::string> lines(android::base::Split(output, "\n"));
97   std::string prefix;
98   std::string suffix;
99   bool capture_prefix = true;
100   for (auto iter = lines.begin(); iter != lines.end(); ++iter) {
101     const char* line = iter->c_str();
102     if (line[0] == '\x1B') {
103       // Skip the escape sequence.
104       line = &line[7];
105     }
106     if (strncmp(line, "[ RUN      ]", 12) == 0) {
107       std::string test_name(iter->substr(12));
108       std::string test_body(*iter + '\n');
109       bool ended = false;
110       for (++iter; iter != lines.end(); ++iter) {
111         test_body += *iter + '\n';
112         line = iter->c_str();
113         if (line[0] == '\x1B') {
114           // Skip the escape sequence.
115           line = &line[7];
116         }
117         if (strncmp(line, "[ ", 2) == 0) {
118           ended = true;
119           break;
120         }
121       }
122       if (!ended) {
123         return output;
124       }
125       tests[test_name] = test_body;
126       capture_prefix = false;
127     } else if (capture_prefix) {
128       prefix += *iter + '\n';
129     } else {
130       suffix += *iter + '\n';
131     }
132   }
133 
134   std::string new_output("Output Sorted\n" + prefix);
135   for (auto entry : tests) {
136     new_output += entry.second;
137   }
138   new_output += suffix;
139   if (android::base::EndsWith(new_output, "\n\n")) {
140     new_output.resize(new_output.size() - 1);
141   }
142   return new_output;
143 }
144 
SanitizeOutput()145 void SystemTests::SanitizeOutput() {
146   // Change (100 ms to (XX ms
147   sanitized_output_ =
148       std::regex_replace(raw_output_, std::regex("\\(\\d+ ms(\\)|\\s|,)"), "(XX ms$1");
149 
150   // Change (elapsed time 100 ms to (elapsed time XX ms
151   sanitized_output_ = std::regex_replace(
152       sanitized_output_, std::regex("\\(elapsed time \\d+ ms(\\)|\\s|,)"), "(elapsed time XX ms$1");
153 
154   // Change stopped|timeout at 100 ms to stopped|timeout at XX ms
155   sanitized_output_ = std::regex_replace(sanitized_output_,
156                                          std::regex("(stopped|timeout) at \\d+ ms"), "$1 at XX ms");
157 
158   // Change any error message like .../file.cc:(200) to file:(XX)
159   sanitized_output_ = std::regex_replace(
160       sanitized_output_, std::regex("\\b([^/\\s]+/)*[^/\\s]+:\\(\\d+\\)\\s"), "file:(XX) ");
161 
162   // Change any terminated by signal message to ignore the actual signal name.
163   sanitized_output_ =
164       std::regex_replace(sanitized_output_, std::regex("( terminated by signal:) .*"), "$1 XXX");
165 }
166 
Exec(std::vector<const char * > args)167 void SystemTests::Exec(std::vector<const char*> args) {
168   int fds[2];
169 #if !defined(__APPLE__)
170   ASSERT_NE(-1, pipe2(fds, O_NONBLOCK));
171 #else
172   ASSERT_NE(-1, pipe(fds));
173   ASSERT_NE(-1, fcntl(fds[0], F_SETFL, O_NONBLOCK));
174 #endif
175 
176   if ((pid_ = fork()) == 0) {
177     // Run the test.
178     close(fds[0]);
179     close(STDIN_FILENO);
180     close(STDOUT_FILENO);
181     close(STDERR_FILENO);
182     ASSERT_NE(0, dup2(fds[1], STDOUT_FILENO));
183     ASSERT_NE(0, dup2(fds[1], STDERR_FILENO));
184     close(fds[1]);
185 
186     std::string exe_name = android::base::GetExecutablePath();
187     args.insert(args.begin(), exe_name.c_str());
188     args.push_back(nullptr);
189     execv(args[0], reinterpret_cast<char* const*>(const_cast<char**>(args.data())));
190     exit(1);
191   }
192   ASSERT_NE(-1, pid_);
193 
194   close(fds[1]);
195   fd_ = fds[0];
196 }
197 
ExecAndCapture(std::vector<const char * > args)198 void SystemTests::ExecAndCapture(std::vector<const char*> args) {
199   Exec(args);
200 
201   int flags = fcntl(fd_, F_GETFL);
202   ASSERT_NE(-1, flags);
203   flags &= ~O_NONBLOCK;
204   ASSERT_NE(-1, fcntl(fd_, F_SETFL, flags));
205   ASSERT_TRUE(android::base::ReadFdToString(fd_, &raw_output_));
206   close(fd_);
207 
208   int status;
209   ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
210   exitcode_ = WEXITSTATUS(status);
211   SanitizeOutput();
212 }
213 
RunTest(const std::string & test_name,std::vector<const char * > extra_args)214 void SystemTests::RunTest(const std::string& test_name, std::vector<const char*> extra_args) {
215   std::vector<const char*> args;
216   bool job_count = false;
217   for (const auto& arg : extra_args) {
218     if (strncmp(arg, "-j", 2) == 0) {
219       job_count = true;
220     }
221     args.push_back(arg);
222   }
223   if (!job_count) {
224     // Always set to only 20 jobs if no job count option is set.
225     args.push_back("-j20");
226   }
227   args.push_back("--gtest_also_run_disabled_tests");
228   std::string filter_arg("--gtest_filter=" + test_name);
229   args.push_back(filter_arg.c_str());
230 
231   ExecAndCapture(args);
232 }
233 
RunTestCaptureFooter(const std::string & test_name,std::string * footer,std::vector<const char * > extra_args)234 void SystemTests::RunTestCaptureFooter(const std::string& test_name, std::string* footer,
235                                        std::vector<const char*> extra_args) {
236   ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
237 
238   // Verify the order of the output messages.
239   std::stringstream stream(sanitized_output_);
240   std::string line;
241   footer->clear();
242   while (std::getline(stream, line, '\n')) {
243     if (!footer->empty()) {
244       *footer += line + '\n';
245     } else if (android::base::StartsWith(line, "[  PASSED  ] ")) {
246       *footer = line + '\n';
247     }
248   }
249   ASSERT_FALSE(footer->empty()) << "Test output:\n" << raw_output_;
250 }
251 
Verify(const std::string & test_name,const std::string & expected_output,int expected_exitcode,std::vector<const char * > extra_args)252 void SystemTests::Verify(const std::string& test_name, const std::string& expected_output,
253                          int expected_exitcode, std::vector<const char*> extra_args) {
254   ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
255   ASSERT_EQ(expected_exitcode, exitcode_) << "Test output:\n" << raw_output_;
256   if (!expected_output.empty()) {
257     ASSERT_EQ(expected_output, sanitized_output_);
258   }
259 }
260 
VerifySortedOutput(const std::string & test_name,const std::string & expected_output,int expected_exitcode,std::vector<const char * > extra_args)261 void SystemTests::VerifySortedOutput(const std::string& test_name,
262                                      const std::string& expected_output, int expected_exitcode,
263                                      std::vector<const char*> extra_args) {
264   ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
265   ASSERT_EQ(expected_exitcode, exitcode_) << "Test output:\n" << raw_output_;
266   if (!expected_output.empty()) {
267     std::string sorted_output = SortTestOutput(sanitized_output_);
268     ASSERT_EQ(expected_output, sorted_output);
269   }
270 }
271 
TEST_F(SystemTests,verify_pass)272 TEST_F(SystemTests, verify_pass) {
273   std::string expected =
274       "Note: Google Test filter = *.DISABLED_pass\n"
275       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
276       "[ RUN      ] SystemTests.DISABLED_pass\n"
277       "[       OK ] SystemTests.DISABLED_pass (XX ms)\n"
278       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
279       "[  PASSED  ] 1 test.\n";
280   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_pass", expected, 0));
281 }
282 
TEST_F(SystemTests,verify_pass_no_print_time)283 TEST_F(SystemTests, verify_pass_no_print_time) {
284   std::string expected =
285       "Note: Google Test filter = *.DISABLED_pass\n"
286       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
287       "[ RUN      ] SystemTests.DISABLED_pass\n"
288       "[       OK ] SystemTests.DISABLED_pass\n"
289       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
290       "[  PASSED  ] 1 test.\n";
291   ASSERT_NO_FATAL_FAILURE(
292       Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{"--gtest_print_time=0"}));
293 }
294 
TEST_F(SystemTests,verify_pass_color)295 TEST_F(SystemTests, verify_pass_color) {
296   std::string expected =
297       "\x1B[0;33mNote: Google Test filter = *.DISABLED_pass\x1B[m\n"
298       "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
299       "\x1B[0;32m[ RUN      ]\x1B[m SystemTests.DISABLED_pass\n"
300       "\x1B[0;32m[       OK ]\x1B[m SystemTests.DISABLED_pass (XX ms)\n"
301       "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
302       "\x1B[0;32m[  PASSED  ]\x1B[m 1 test.\n";
303   ASSERT_NO_FATAL_FAILURE(
304       Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{"--gtest_color=yes"}));
305 }
306 
TEST_F(SystemTests,verify_skip)307 TEST_F(SystemTests, verify_skip) {
308   std::string expected =
309       "Note: Google Test filter = *.DISABLED_skip_no_message\n"
310       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
311       "[ RUN      ] SystemTests.DISABLED_skip_no_message\n"
312       "file:(XX) Skipped\n"
313       "[  SKIPPED ] SystemTests.DISABLED_skip_no_message (XX ms)\n"
314       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
315       "[  PASSED  ] 0 tests.\n"
316       "[  SKIPPED ] 1 test, listed below:\n"
317       "[  SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
318   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0));
319 }
320 
TEST_F(SystemTests,verify_skip_with_message)321 TEST_F(SystemTests, verify_skip_with_message) {
322   std::string expected =
323       "Note: Google Test filter = *.DISABLED_skip_with_message\n"
324       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
325       "[ RUN      ] SystemTests.DISABLED_skip_with_message\n"
326       "file:(XX) Skipped\n"
327       "This is a skip message\n"
328       "[  SKIPPED ] SystemTests.DISABLED_skip_with_message (XX ms)\n"
329       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
330       "[  PASSED  ] 0 tests.\n"
331       "[  SKIPPED ] 1 test, listed below:\n"
332       "[  SKIPPED ] SystemTests.DISABLED_skip_with_message\n";
333   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_message", expected, 0));
334 }
335 
TEST_F(SystemTests,verify_skip_with_output_before_message)336 TEST_F(SystemTests, verify_skip_with_output_before_message) {
337   std::string expected =
338       "Note: Google Test filter = *.DISABLED_skip_with_output_before\n"
339       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
340       "[ RUN      ] SystemTests.DISABLED_skip_with_output_before\n"
341       "This is the message before the skip message\n"
342       "file:(XX) Skipped\n"
343       "This is the skip message\n"
344       "[  SKIPPED ] SystemTests.DISABLED_skip_with_output_before (XX ms)\n"
345       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
346       "[  PASSED  ] 0 tests.\n"
347       "[  SKIPPED ] 1 test, listed below:\n"
348       "[  SKIPPED ] SystemTests.DISABLED_skip_with_output_before\n";
349   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_output_before", expected, 0));
350 }
351 
TEST_F(SystemTests,verify_skip_with_output_after_message)352 TEST_F(SystemTests, verify_skip_with_output_after_message) {
353   std::string expected =
354       "Note: Google Test filter = *.DISABLED_skip_with_output_after\n"
355       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
356       "[ RUN      ] SystemTests.DISABLED_skip_with_output_after\n"
357       "file:(XX) Skipped\n"
358       "This is the skip message\n"
359       "This is the message after the skip message\n"
360       "[  SKIPPED ] SystemTests.DISABLED_skip_with_output_after (XX ms)\n"
361       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
362       "[  PASSED  ] 0 tests.\n"
363       "[  SKIPPED ] 1 test, listed below:\n"
364       "[  SKIPPED ] SystemTests.DISABLED_skip_with_output_after\n";
365   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_output_after", expected, 0));
366 }
367 
TEST_F(SystemTests,verify_skip_with_skipped_line)368 TEST_F(SystemTests, verify_skip_with_skipped_line) {
369   std::string expected =
370       "Note: Google Test filter = *.DISABLED_skip_with_skipped_line\n"
371       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
372       "[ RUN      ] SystemTests.DISABLED_skip_with_skipped_line\n"
373       "\n"
374       "Skipped\n"
375       "file:(XX) Skipped\n"
376       "This is the skip message 1\n"
377       "Skipped\n"
378       "file:(XX) Skipped\n"
379       "This is the skip message 2\n"
380       "Skipped\n"
381       "[  SKIPPED ] SystemTests.DISABLED_skip_with_skipped_line (XX ms)\n"
382       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
383       "[  PASSED  ] 0 tests.\n"
384       "[  SKIPPED ] 1 test, listed below:\n"
385       "[  SKIPPED ] SystemTests.DISABLED_skip_with_skipped_line\n";
386   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_skipped_line", expected, 0));
387 }
388 
TEST_F(SystemTests,verify_skip_multiple)389 TEST_F(SystemTests, verify_skip_multiple) {
390   std::string expected =
391       "Note: Google Test filter = *.DISABLED_skip_multiple\n"
392       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
393       "[ RUN      ] SystemTests.DISABLED_skip_multiple\n"
394       "This is not a skip message 1\n"
395       "file:(XX) Skipped\n"
396       "This is the skip message 1\n"
397       "This is not a skip message 2\n"
398       "file:(XX) Skipped\n"
399       "This is the skip message 2\n"
400       "file:(XX) Skipped\n"
401       "This is the skip message 3\n"
402       "This is not a skip message 4\n"
403       "[  SKIPPED ] SystemTests.DISABLED_skip_multiple (XX ms)\n"
404       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
405       "[  PASSED  ] 0 tests.\n"
406       "[  SKIPPED ] 1 test, listed below:\n"
407       "[  SKIPPED ] SystemTests.DISABLED_skip_multiple\n";
408   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_multiple", expected, 0));
409 }
410 
TEST_F(SystemTests,verify_skip_no_print_time)411 TEST_F(SystemTests, verify_skip_no_print_time) {
412   std::string expected =
413       "Note: Google Test filter = *.DISABLED_skip_no_message\n"
414       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
415       "[ RUN      ] SystemTests.DISABLED_skip_no_message\n"
416       "file:(XX) Skipped\n"
417       "[  SKIPPED ] SystemTests.DISABLED_skip_no_message\n"
418       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
419       "[  PASSED  ] 0 tests.\n"
420       "[  SKIPPED ] 1 test, listed below:\n"
421       "[  SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
422   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
423                                  std::vector<const char*>{"--gtest_print_time=0"}));
424 }
425 
TEST_F(SystemTests,verify_skip_color)426 TEST_F(SystemTests, verify_skip_color) {
427   std::string expected =
428       "\x1B[0;33mNote: Google Test filter = *.DISABLED_skip_no_message\x1B[m\n"
429       "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
430       "\x1B[0;32m[ RUN      ]\x1B[m SystemTests.DISABLED_skip_no_message\n"
431       "file:(XX) Skipped\n"
432       "\x1B[0;32m[  SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message (XX ms)\n"
433       "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
434       "\x1B[0;32m[  PASSED  ]\x1B[m 0 tests.\n"
435       "\x1B[0;32m[  SKIPPED ]\x1B[m 1 test, listed below:\n"
436       "\x1B[0;32m[  SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message\n";
437   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
438                                  std::vector<const char*>{"--gtest_color=yes"}));
439 }
440 
TEST_F(SystemTests,verify_xfail_fail_expect_to_fail)441 TEST_F(SystemTests, verify_xfail_fail_expect_to_fail) {
442   std::string expected =
443       "Note: Google Test filter = *.xfail_fail\n"
444       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
445       "[ RUN      ] DISABLED_SystemTestsXfail.xfail_fail\n"
446       "file:(XX) Failure in test DISABLED_SystemTestsXfail.xfail_fail\n"
447       "Expected equality of these values:\n"
448       "  1\n"
449       "  0\n"
450       "DISABLED_SystemTestsXfail.xfail_fail exited with exitcode 1.\n"
451       "[       OK ] DISABLED_SystemTestsXfail.xfail_fail (XX ms)\n"
452       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
453       "[  PASSED  ] 1 test. (1 expected failure)\n";
454   ASSERT_NO_FATAL_FAILURE(Verify("*.xfail_fail", expected, 0));
455 }
456 
TEST_F(SystemTests,verify_xfail_pass_expect_to_fail)457 TEST_F(SystemTests, verify_xfail_pass_expect_to_fail) {
458   std::string expected =
459       "Note: Google Test filter = *.xfail_pass\n"
460       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
461       "[ RUN      ] DISABLED_SystemTestsXfail.xfail_pass\n"
462       "[  FAILED  ] DISABLED_SystemTestsXfail.xfail_pass (XX ms)\n"
463       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
464       "[  PASSED  ] 0 tests.\n"
465       "[  FAILED  ] 1 test should have failed, listed below:\n"
466       "[  FAILED  ] DISABLED_SystemTestsXfail.xfail_pass\n"
467       "\n"
468       " 1 SHOULD HAVE FAILED TEST\n";
469   ASSERT_NO_FATAL_FAILURE(Verify("*.xfail_pass", expected, 1));
470 }
471 
TEST_F(SystemTests,verify_xfail_pass_expect_to_fail_color)472 TEST_F(SystemTests, verify_xfail_pass_expect_to_fail_color) {
473   std::string expected =
474       "\x1B[0;33mNote: Google Test filter = *.xfail_pass\x1B[m\n"
475       "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
476       "\x1B[0;32m[ RUN      ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass\n"
477       "\x1B[0;31m[  FAILED  ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass (XX ms)\n"
478       "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
479       "\x1B[0;32m[  PASSED  ]\x1B[m 0 tests.\n"
480       "\x1B[0;31m[  FAILED  ]\x1B[m 1 test should have failed, listed below:\n"
481       "\x1B[0;31m[  FAILED  ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass\n"
482       "\n"
483       " 1 SHOULD HAVE FAILED TEST\n";
484   ASSERT_NO_FATAL_FAILURE(
485       Verify("*.xfail_pass", expected, 1, std::vector<const char*>{"--gtest_color=yes"}));
486 }
487 
TEST_F(SystemTests,verify_deathtest_pass)488 TEST_F(SystemTests, verify_deathtest_pass) {
489   std::string expected =
490       "Note: Google Test filter = *.DISABLED_death_pass\n"
491       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
492       "[ RUN      ] SystemTestsDeathTest.DISABLED_death_pass\n"
493       "[       OK ] SystemTestsDeathTest.DISABLED_death_pass (XX ms)\n"
494       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
495       "[  PASSED  ] 1 test.\n";
496   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_death_pass", expected, 0));
497 }
498 
TEST_F(SystemTests,verify_fail)499 TEST_F(SystemTests, verify_fail) {
500   std::string expected =
501       "Note: Google Test filter = *.DISABLED_fail\n"
502       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
503       "[ RUN      ] SystemTests.DISABLED_fail\n"
504       "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
505       "Expected equality of these values:\n"
506       "  1\n"
507       "  0\n"
508       "SystemTests.DISABLED_fail exited with exitcode 1.\n"
509       "[  FAILED  ] SystemTests.DISABLED_fail (XX ms)\n"
510       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
511       "[  PASSED  ] 0 tests.\n"
512       "[  FAILED  ] 1 test, listed below:\n"
513       "[  FAILED  ] SystemTests.DISABLED_fail\n"
514       "\n"
515       " 1 FAILED TEST\n";
516   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_fail", expected, 1));
517 }
518 
TEST_F(SystemTests,verify_fail_color)519 TEST_F(SystemTests, verify_fail_color) {
520   std::string expected =
521       "\x1B[0;33mNote: Google Test filter = *.DISABLED_fail\x1B[m\n"
522       "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
523       "\x1B[0;32m[ RUN      ]\x1B[m SystemTests.DISABLED_fail\n"
524       "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
525       "Expected equality of these values:\n"
526       "  1\n"
527       "  0\n"
528       "SystemTests.DISABLED_fail exited with exitcode 1.\n"
529       "\x1B[0;31m[  FAILED  ]\x1B[m SystemTests.DISABLED_fail (XX ms)\n"
530       "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
531       "\x1B[0;32m[  PASSED  ]\x1B[m 0 tests.\n"
532       "\x1B[0;31m[  FAILED  ]\x1B[m 1 test, listed below:\n"
533       "\x1B[0;31m[  FAILED  ]\x1B[m SystemTests.DISABLED_fail\n"
534       "\n"
535       " 1 FAILED TEST\n";
536   ASSERT_NO_FATAL_FAILURE(
537       Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_color=yes"}));
538 }
539 
TEST_F(SystemTests,verify_deathtest_fail)540 TEST_F(SystemTests, verify_deathtest_fail) {
541   std::string expected =
542       "Note: Google Test filter = *.DISABLED_death_fail\n"
543       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
544       "[ RUN      ] SystemTestsDeathTest.DISABLED_death_fail\n"
545       "file:(XX) Failure in test SystemTestsDeathTest.DISABLED_death_fail\n"
546       "Death test: DeathTestHelperFail()\n"
547       "    Result: failed to die.\n"
548       " Error msg:\n"
549       "[  DEATH   ] \n"
550       "SystemTestsDeathTest.DISABLED_death_fail exited with exitcode 1.\n"
551       "[  FAILED  ] SystemTestsDeathTest.DISABLED_death_fail (XX ms)\n"
552       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
553       "[  PASSED  ] 0 tests.\n"
554       "[  FAILED  ] 1 test, listed below:\n"
555       "[  FAILED  ] SystemTestsDeathTest.DISABLED_death_fail\n"
556       "\n"
557       " 1 FAILED TEST\n";
558   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_death_fail", expected, 1));
559 }
560 
TEST_F(SystemTests,verify_crash)561 TEST_F(SystemTests, verify_crash) {
562   std::string expected =
563       "Note: Google Test filter = *.DISABLED_crash\n"
564       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
565       "[ RUN      ] SystemTests.DISABLED_crash\n"
566       "SystemTests.DISABLED_crash terminated by signal: XXX\n"
567       "[  FAILED  ] SystemTests.DISABLED_crash (XX ms)\n"
568       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
569       "[  PASSED  ] 0 tests.\n"
570       "[  FAILED  ] 1 test, listed below:\n"
571       "[  FAILED  ] SystemTests.DISABLED_crash\n"
572       "\n"
573       " 1 FAILED TEST\n";
574   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_crash", expected, 1));
575 }
576 
TEST_F(SystemTests,verify_warning_slow)577 TEST_F(SystemTests, verify_warning_slow) {
578   std::string expected =
579       "Note: Google Test filter = *.DISABLED_sleep5\n"
580       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
581       "[ RUN      ] SystemTests.DISABLED_sleep5\n"
582       "[       OK ] SystemTests.DISABLED_sleep5 (XX ms)\n"
583       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
584       "[  PASSED  ] 1 test.\n"
585       "[  SLOW    ] 1 test, listed below:\n"
586       "[  SLOW    ] SystemTests.DISABLED_sleep5 (XX ms, exceeded 3000 ms)\n"
587       "\n"
588       " 1 SLOW TEST\n";
589   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_sleep5", expected, 0,
590                                  std::vector<const char*>{"--slow_threshold_ms=3000"}));
591 }
592 
TEST_F(SystemTests,verify_warning_slow_color)593 TEST_F(SystemTests, verify_warning_slow_color) {
594   std::string expected =
595       "\x1B[0;33mNote: Google Test filter = *.DISABLED_sleep5\x1B[m\n"
596       "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
597       "\x1B[0;32m[ RUN      ]\x1B[m SystemTests.DISABLED_sleep5\n"
598       "\x1B[0;32m[       OK ]\x1B[m SystemTests.DISABLED_sleep5 (XX ms)\n"
599       "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
600       "\x1B[0;32m[  PASSED  ]\x1B[m 1 test.\n"
601       "\x1B[0;33m[  SLOW    ]\x1B[m 1 test, listed below:\n"
602       "\x1B[0;33m[  SLOW    ]\x1B[m SystemTests.DISABLED_sleep5 (XX ms, exceeded 3000 ms)\n"
603       "\n"
604       " 1 SLOW TEST\n";
605   ASSERT_NO_FATAL_FAILURE(
606       Verify("*.DISABLED_sleep5", expected, 0,
607              std::vector<const char*>{"--slow_threshold_ms=3000", "--gtest_color=yes"}));
608 }
609 
TEST_F(SystemTests,verify_timeout)610 TEST_F(SystemTests, verify_timeout) {
611   std::string expected =
612       "Note: Google Test filter = *.DISABLED_sleep_forever\n"
613       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
614       "[ RUN      ] SystemTests.DISABLED_sleep_forever\n"
615       "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
616       "[  FAILED  ] SystemTests.DISABLED_sleep_forever (XX ms)\n"
617       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
618       "[  PASSED  ] 0 tests.\n"
619       "[  TIMEOUT ] 1 test, listed below:\n"
620       "[  TIMEOUT ] SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
621       "\n"
622       " 1 TIMEOUT TEST\n";
623   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_sleep_forever", expected, 1,
624                                  std::vector<const char*>{"--deadline_threshold_ms=3000"}));
625 }
626 
627 // Verify that tests that timeout do not get marked as slow too when
628 // another test is marked as slow.
TEST_F(SystemTests,verify_timeout_not_slow)629 TEST_F(SystemTests, verify_timeout_not_slow) {
630   std::string expected =
631       "Note: Google Test filter = *.DISABLED_sleep*\n"
632       "[==========] Running 2 tests from 1 test suite (20 jobs).\n"
633       "[ RUN      ] SystemTests.DISABLED_sleep5\n"
634       "[       OK ] SystemTests.DISABLED_sleep5 (XX ms)\n"
635       "[ RUN      ] SystemTests.DISABLED_sleep_forever\n"
636       "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
637       "[  FAILED  ] SystemTests.DISABLED_sleep_forever (XX ms)\n"
638       "[==========] 2 tests from 1 test suite ran. (XX ms total)\n"
639       "[  PASSED  ] 1 test.\n"
640       "[  SLOW    ] 1 test, listed below:\n"
641       "[  SLOW    ] SystemTests.DISABLED_sleep5 (XX ms, exceeded 1000 ms)\n"
642       "[  TIMEOUT ] 1 test, listed below:\n"
643       "[  TIMEOUT ] SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
644       "\n"
645       " 1 SLOW TEST\n"
646       " 1 TIMEOUT TEST\n";
647   ASSERT_NO_FATAL_FAILURE(Verify(
648       "*.DISABLED_sleep*", expected, 1,
649       std::vector<const char*>{"--slow_threshold_ms=1000", "--deadline_threshold_ms=10000"}));
650 }
651 
TEST_F(SystemTests,verify_timeout_color)652 TEST_F(SystemTests, verify_timeout_color) {
653   std::string expected =
654       "\x1B[0;33mNote: Google Test filter = *.DISABLED_sleep_forever\x1B[m\n"
655       "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
656       "\x1B[0;32m[ RUN      ]\x1B[m SystemTests.DISABLED_sleep_forever\n"
657       "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
658       "\x1B[0;31m[  FAILED  ]\x1B[m SystemTests.DISABLED_sleep_forever (XX ms)\n"
659       "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
660       "\x1B[0;32m[  PASSED  ]\x1B[m 0 tests.\n"
661       "\x1B[0;31m[  TIMEOUT ]\x1B[m 1 test, listed below:\n"
662       "\x1B[0;31m[  TIMEOUT ]\x1B[m SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
663       "\n"
664       " 1 TIMEOUT TEST\n";
665   ASSERT_NO_FATAL_FAILURE(
666       Verify("*.DISABLED_sleep_forever", expected, 1,
667              std::vector<const char*>{"--deadline_threshold_ms=3000", "--gtest_color=yes"}));
668 }
669 
TEST_F(SystemTests,verify_order_isolated)670 TEST_F(SystemTests, verify_order_isolated) {
671   std::string expected =
672       "Note: Google Test filter = *.DISABLED_order_*\n"
673       "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
674       "[ RUN      ] SystemTests.DISABLED_order_3\n"
675       "[       OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
676       "[ RUN      ] SystemTests.DISABLED_order_2\n"
677       "[       OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
678       "[ RUN      ] SystemTests.DISABLED_order_1\n"
679       "[       OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
680       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
681       "[  PASSED  ] 3 tests.\n";
682   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_order_*", expected, 0));
683 }
684 
TEST_F(SystemTests,verify_order_not_isolated)685 TEST_F(SystemTests, verify_order_not_isolated) {
686   std::string expected =
687       "Note: Google Test filter = *.DISABLED_order_*\n"
688       "[==========] Running 3 tests from 1 test suite.\n"
689       "[----------] Global test environment set-up.\n"
690       "[----------] 3 tests from SystemTests\n"
691       "[ RUN      ] SystemTests.DISABLED_order_1\n"
692       "[       OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
693       "[ RUN      ] SystemTests.DISABLED_order_2\n"
694       "[       OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
695       "[ RUN      ] SystemTests.DISABLED_order_3\n"
696       "[       OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
697       "[----------] 3 tests from SystemTests (XX ms total)\n"
698       "\n"
699       "[----------] Global test environment tear-down\n"
700       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
701       "[  PASSED  ] 3 tests.\n";
702   ASSERT_NO_FATAL_FAILURE(
703       Verify("*.DISABLED_order_*", expected, 0, std::vector<const char*>{"--no_isolate"}));
704 }
705 
TEST_F(SystemTests,verify_fail_ge10)706 TEST_F(SystemTests, verify_fail_ge10) {
707   ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_fail_*"));
708   // Verify the failed output at the end has no space in front.
709   std::regex regex("\\n.*\\d+ FAILED TESTS\\n");
710   std::cmatch match;
711   ASSERT_TRUE(std::regex_search(sanitized_output_.c_str(), match, regex)) << "Test Output:\n"
712                                                                           << raw_output_;
713   ASSERT_EQ("\n10 FAILED TESTS\n", match[0]);
714   ASSERT_NE(0, exitcode_);
715 }
716 
TEST_F(SystemTests,verify_title_order)717 TEST_F(SystemTests, verify_title_order) {
718   std::string footer;
719   ASSERT_NO_FATAL_FAILURE(RunTestCaptureFooter(
720       "*.DISABLED_all_*", &footer,
721       std::vector<const char*>{"--slow_threshold_ms=2000", "--deadline_threshold_ms=4000"}));
722 
723   ASSERT_EQ(
724       "[  PASSED  ] 4 tests.\n"
725       "[  SKIPPED ] 2 tests, listed below:\n"
726       "[  SKIPPED ] SystemTests.DISABLED_all_skip_1\n"
727       "[  SKIPPED ] SystemTests.DISABLED_all_skip_2\n"
728       "[  SLOW    ] 2 tests, listed below:\n"
729       "[  SLOW    ] SystemTests.DISABLED_all_slow_1 (XX ms, exceeded 2000 ms)\n"
730       "[  SLOW    ] SystemTests.DISABLED_all_slow_2 (XX ms, exceeded 2000 ms)\n"
731       "[  TIMEOUT ] 2 tests, listed below:\n"
732       "[  TIMEOUT ] SystemTests.DISABLED_all_timeout_1 (stopped at XX ms)\n"
733       "[  TIMEOUT ] SystemTests.DISABLED_all_timeout_2 (stopped at XX ms)\n"
734       "[  FAILED  ] 2 tests, listed below:\n"
735       "[  FAILED  ] SystemTests.DISABLED_all_fail_1\n"
736       "[  FAILED  ] SystemTests.DISABLED_all_fail_2\n"
737       "\n"
738       " 2 SLOW TESTS\n"
739       " 2 TIMEOUT TESTS\n"
740       " 2 FAILED TESTS\n",
741       footer);
742 }
743 
TEST_F(SystemTests,verify_job_count_single)744 TEST_F(SystemTests, verify_job_count_single) {
745   std::string expected =
746       "Note: Google Test filter = *.DISABLED_job_*\n"
747       "[==========] Running 3 tests from 1 test suite (1 job).\n"
748       "[ RUN      ] SystemTests.DISABLED_job_1\n"
749       "[       OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
750       "[ RUN      ] SystemTests.DISABLED_job_2\n"
751       "[       OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
752       "[ RUN      ] SystemTests.DISABLED_job_3\n"
753       "[       OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
754       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
755       "[  PASSED  ] 3 tests.\n";
756   ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_job_*", expected, 0, std::vector<const char*>{"-j1"}));
757 }
758 
TEST_F(SystemTests,verify_job_count_multiple)759 TEST_F(SystemTests, verify_job_count_multiple) {
760   std::string expected =
761       "Note: Google Test filter = *.DISABLED_job_*\n"
762       "[==========] Running 3 tests from 1 test suite (2 jobs).\n"
763       "[ RUN      ] SystemTests.DISABLED_job_2\n"
764       "[       OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
765       "[ RUN      ] SystemTests.DISABLED_job_1\n"
766       "[       OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
767       "[ RUN      ] SystemTests.DISABLED_job_3\n"
768       "[       OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
769       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
770       "[  PASSED  ] 3 tests.\n";
771   ASSERT_NO_FATAL_FAILURE(
772       Verify("*.DISABLED_job_*", expected, 0, std::vector<const char*>{"-j", "2"}));
773 }
774 
TEST_F(SystemTests,verify_help)775 TEST_F(SystemTests, verify_help) {
776   // This tests verifies that the help options display the help for
777   // the isolated test run, and for the gtest data.
778   std::vector<const char*> help_args{"-h", "--help"};
779   for (auto arg : help_args) {
780     ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_pass", std::vector<const char*>{arg}));
781     ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
782     // First find something from the isolation help.
783     std::size_t isolation_help = sanitized_output_.find("In isolation mode,");
784     ASSERT_NE(std::string::npos, isolation_help) << "Cannot find isolation help:\n" << raw_output_;
785     std::size_t gtest_help = sanitized_output_.find("Assertion Behavior:");
786     ASSERT_NE(std::string::npos, gtest_help) << "Cannot find gtest help:\n" << raw_output_;
787 
788     ASSERT_GT(gtest_help, isolation_help) << "Gtest help before isolation help:\n" << raw_output_;
789   }
790 }
791 
TEST_F(SystemTests,verify_help_color)792 TEST_F(SystemTests, verify_help_color) {
793   // Verify that the color option does change the help display.
794   std::vector<const char*> help_args{"-h", "--help"};
795   for (auto arg : help_args) {
796     ASSERT_NO_FATAL_FAILURE(
797         RunTest("*.DISABLED_pass", std::vector<const char*>{arg, "--gtest_color=yes"}));
798     ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
799     // First find something from the isolation help that is in color.
800     std::size_t isolation_help =
801         sanitized_output_.find("Unit Test Options:\n\x1B[0;32m  -j \x1B[m");
802     ASSERT_NE(std::string::npos, isolation_help) << "Cannot find isolation help:\n" << raw_output_;
803     std::size_t gtest_help = sanitized_output_.find("\x1B[0;32m--gtest_list_tests\x1B[m");
804     ASSERT_NE(std::string::npos, gtest_help) << "Cannot find gtest help:\n" << raw_output_;
805 
806     ASSERT_GT(gtest_help, isolation_help) << "Gtest help before isolation help:\n" << raw_output_;
807   }
808 }
809 
TEST_F(SystemTests,verify_repeat)810 TEST_F(SystemTests, verify_repeat) {
811   std::string expected =
812       "Note: Google Test filter = *.DISABLED_order_*\n"
813       "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
814       "[ RUN      ] SystemTests.DISABLED_order_3\n"
815       "[       OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
816       "[ RUN      ] SystemTests.DISABLED_order_2\n"
817       "[       OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
818       "[ RUN      ] SystemTests.DISABLED_order_1\n"
819       "[       OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
820       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
821       "[  PASSED  ] 3 tests.\n"
822       "\n"
823       "Repeating all tests (iteration 2) . . .\n"
824       "\n"
825       "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
826       "[ RUN      ] SystemTests.DISABLED_order_3\n"
827       "[       OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
828       "[ RUN      ] SystemTests.DISABLED_order_2\n"
829       "[       OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
830       "[ RUN      ] SystemTests.DISABLED_order_1\n"
831       "[       OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
832       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
833       "[  PASSED  ] 3 tests.\n"
834       "\n"
835       "Repeating all tests (iteration 3) . . .\n"
836       "\n"
837       "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
838       "[ RUN      ] SystemTests.DISABLED_order_3\n"
839       "[       OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
840       "[ RUN      ] SystemTests.DISABLED_order_2\n"
841       "[       OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
842       "[ RUN      ] SystemTests.DISABLED_order_1\n"
843       "[       OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
844       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
845       "[  PASSED  ] 3 tests.\n";
846   uint64_t time_ns = NanoTime();
847   ASSERT_NO_FATAL_FAILURE(
848       Verify("*.DISABLED_order_*", expected, 0, std::vector<const char*>{"--gtest_repeat=3"}));
849   time_ns = NanoTime() - time_ns;
850   // Make sure that the total test time is about 18 seconds.
851   double seconds = double(time_ns) / 1000000000;
852   ASSERT_LE(18.0, seconds) << "Repeat test should take at least 18 seconds.\n"
853                            << "Test output:\n"
854                            << raw_output_;
855   ASSERT_GT(20.0, seconds) << "Repeat test should take about 18 seconds.\n"
856                            << "Test output:\n"
857                            << raw_output_;
858 }
859 
TEST_F(SystemTests,verify_results_as_tests_finish)860 TEST_F(SystemTests, verify_results_as_tests_finish) {
861   // This test verifies that test output comes out as the test finishes.
862   Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_order_*",
863                                 "--gtest_also_run_disabled_tests", "-j20"});
864 
865   std::string output;
866   std::vector<char> buffer(4096);
867   uint64_t time_ns = NanoTime();
868   while (true) {
869     ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
870     if (bytes == -1 && errno == EAGAIN) {
871       continue;
872     }
873     ASSERT_NE(-1, bytes);
874     ASSERT_NE(0, bytes) << "Did not find test output before test finished:\n" << output;
875     buffer[bytes] = '\0';
876     output += buffer.data();
877     // See if the output has come out now.
878     if (output.find("[       OK ] SystemTests.DISABLED_order_2") != std::string::npos) {
879       uint64_t test_ns = NanoTime() - time_ns;
880       double test_sec = double(test_ns) / 1000000000;
881       // This should happen after 3 seconds, but before 4.5 seconds.
882       ASSERT_LE(3.0, test_sec) << "Test output:\n" << output;
883       ASSERT_GT(4.5, test_sec) << "Test output:\n" << output;
884       break;
885     }
886   }
887 
888   // Read the rest of the output.
889   while (true) {
890     ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
891     if (bytes == -1 && errno == EAGAIN) {
892       continue;
893     }
894     ASSERT_NE(-1, bytes);
895     if (bytes == 0) {
896       break;
897     }
898     buffer[bytes] = '\0';
899     output += buffer.data();
900   }
901   close(fd_);
902   time_ns = NanoTime() - time_ns;
903   ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, nullptr, 0))) << "Test output:\n" << output;
904   // Verify that the total test time is > 6 seconds.
905   ASSERT_LE(6.0, double(time_ns) / 1000000000) << "Test output:\n" << output;
906 }
907 
TEST_F(SystemTests,verify_xml)908 TEST_F(SystemTests, verify_xml) {
909   std::string tmp_arg("--gtest_output=xml:");
910   TemporaryFile tf;
911   ASSERT_TRUE(tf.fd != -1);
912   close(tf.fd);
913   tmp_arg += tf.path;
914 
915   ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_xml_*", std::vector<const char*>{tmp_arg.c_str()}));
916   ASSERT_EQ(1, exitcode_) << "Test output:\n" << raw_output_;
917 
918   // Check that the xml file exists.
919   FILE* xml_file = fopen(tf.path, "r");
920   ASSERT_TRUE(xml_file != nullptr) << "Failed to find xml file:\n" << raw_output_;
921   // Read the entire file in.
922   std::string xml_output;
923   std::vector<char> buffer(4096);
924   size_t bytes;
925   while ((bytes = fread(buffer.data(), 1, buffer.size(), xml_file)) > 0) {
926     xml_output += std::string(buffer.data(), bytes);
927   }
928   fclose(xml_file);
929   unlink(tf.path);
930 
931   // Change time|timestamp="" to time|timestamp="XX"
932   xml_output =
933       std::regex_replace(xml_output, std::regex("(time|timestamp)=\"[^\"]+\""), "$1=\"XX\"");
934   // Change ".*.cc:(XX) to "file:(XX)
935   xml_output = std::regex_replace(xml_output, std::regex("\"([^/\\s]+/)*[^/\\s]+:\\(\\d+\\)\\s"),
936                                   "\"file:(XX) ");
937 
938   std::string expected =
939       "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
940       "<testsuites tests=\"6\" failures=\"3\" disabled=\"0\" errors=\"0\" timestamp=\"XX\" "
941       "time=\"XX\" name=\"AllTests\">\n"
942       "  <testsuite name=\"SystemTestsXml1\" tests=\"2\" failures=\"1\" disabled=\"0\" "
943       "errors=\"0\" time=\"XX\">\n"
944       "    <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
945       "classname=\"SystemTestsXml1\" />\n"
946       "    <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
947       "classname=\"SystemTestsXml1\">\n"
948       "      <failure message=\"file:(XX) Failure in test SystemTestsXml1.DISABLED_xml_2\n"
949       "Expected equality of these values:\n"
950       "  1\n"
951       "  0\n"
952       "SystemTestsXml1.DISABLED_xml_2 exited with exitcode 1.\n"
953       "\" type=\"\">\n"
954       "      </failure>\n"
955       "    </testcase>\n"
956       "  </testsuite>\n"
957       "  <testsuite name=\"SystemTestsXml2\" tests=\"2\" failures=\"1\" disabled=\"0\" "
958       "errors=\"0\" time=\"XX\">\n"
959       "    <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
960       "classname=\"SystemTestsXml2\">\n"
961       "      <failure message=\"file:(XX) Failure in test SystemTestsXml2.DISABLED_xml_1\n"
962       "Expected equality of these values:\n"
963       "  1\n"
964       "  0\n"
965       "SystemTestsXml2.DISABLED_xml_1 exited with exitcode 1.\n"
966       "\" type=\"\">\n"
967       "      </failure>\n"
968       "    </testcase>\n"
969       "    <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
970       "classname=\"SystemTestsXml2\" />\n"
971       "  </testsuite>\n"
972       "  <testsuite name=\"SystemTestsXml3\" tests=\"2\" failures=\"1\" disabled=\"0\" "
973       "errors=\"0\" time=\"XX\">\n"
974       "    <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
975       "classname=\"SystemTestsXml3\" />\n"
976       "    <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
977       "classname=\"SystemTestsXml3\">\n"
978       "      <failure message=\"file:(XX) Failure in test SystemTestsXml3.DISABLED_xml_2\n"
979       "Expected equality of these values:\n"
980       "  1\n"
981       "  0\n"
982       "SystemTestsXml3.DISABLED_xml_2 exited with exitcode 1.\n"
983       "\" type=\"\">\n"
984       "      </failure>\n"
985       "    </testcase>\n"
986       "  </testsuite>\n"
987       "</testsuites>\n";
988   ASSERT_EQ(expected, xml_output);
989 }
990 
TEST_F(SystemTests,verify_disabled_not_displayed_with_no_tests)991 TEST_F(SystemTests, verify_disabled_not_displayed_with_no_tests) {
992   std::vector<const char*> args{"--gtest_filter=NO_TEST_FILTER_MATCH", "-j2"};
993 
994   ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
995   ASSERT_EQ(0, exitcode_);
996   std::string expected =
997       "Note: Google Test filter = NO_TEST_FILTER_MATCH\n"
998       "[==========] Running 0 tests from 0 test suites (2 jobs).\n"
999       "[==========] 0 tests from 0 test suites ran. (XX ms total)\n"
1000       "[  PASSED  ] 0 tests.\n";
1001   ASSERT_EQ(expected, sanitized_output_) << "Test output:\n" << raw_output_;
1002 }
1003 
TEST_F(SystemTests,verify_disabled)1004 TEST_F(SystemTests, verify_disabled) {
1005   std::vector<const char*> args{"--gtest_filter=*always_pass", "-j2"};
1006 
1007   ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1008   ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1009   std::string expected =
1010       "Note: Google Test filter = *always_pass\n"
1011       "[==========] Running 1 test from 1 test suite (2 jobs).\n"
1012       "[ RUN      ] SystemTests.always_pass\n"
1013       "[       OK ] SystemTests.always_pass (XX ms)\n"
1014       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1015       "[  PASSED  ] 1 test.\n"
1016       "\n"
1017       "  YOU HAVE 1 DISABLED TEST\n"
1018       "\n";
1019   ASSERT_EQ(expected, sanitized_output_);
1020 }
1021 
TEST_F(SystemTests,verify_disabled_color)1022 TEST_F(SystemTests, verify_disabled_color) {
1023   std::vector<const char*> args{"--gtest_filter=*always_pass", "-j2", "--gtest_color=yes"};
1024 
1025   ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1026   ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1027   std::string expected =
1028       "\x1B[0;33mNote: Google Test filter = *always_pass\x1B[m\n"
1029       "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (2 jobs).\n"
1030       "\x1B[0;32m[ RUN      ]\x1B[m SystemTests.always_pass\n"
1031       "\x1B[0;32m[       OK ]\x1B[m SystemTests.always_pass (XX ms)\n"
1032       "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
1033       "\x1B[0;32m[  PASSED  ]\x1B[m 1 test.\n"
1034       "\n"
1035       "\x1B[0;33m  YOU HAVE 1 DISABLED TEST\n"
1036       "\n\x1B[m";
1037   ASSERT_EQ(expected, sanitized_output_);
1038 }
1039 
TEST_F(SystemTests,verify_SIGINT)1040 TEST_F(SystemTests, verify_SIGINT) {
1041   // Verify that SIGINT kills all of the tests.
1042   Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_job*", "--gtest_also_run_disabled_tests",
1043                                 "-j20"});
1044   // It is expected that all of the tests will be sleeping so nothing will
1045   // complete by the time the signal is sent.
1046   sleep(1);
1047   ASSERT_NE(-1, kill(pid_, SIGINT));
1048 
1049   std::string output;
1050   std::vector<char> buffer(4096);
1051   while (true) {
1052     ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1053     if (bytes == -1 && errno == EAGAIN) {
1054       continue;
1055     }
1056     ASSERT_NE(-1, bytes);
1057     if (bytes == 0) {
1058       break;
1059     }
1060     buffer[bytes] = '\0';
1061     output += buffer.data();
1062   }
1063   close(fd_);
1064   int status;
1065   ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << output;
1066   ASSERT_EQ(
1067       "Note: Google Test filter = *.DISABLED_job*\n"
1068       "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1069       "Terminating due to signal...\n",
1070       output);
1071   ASSERT_EQ(1, WEXITSTATUS(status));
1072 }
1073 
TEST_F(SystemTests,verify_SIGQUIT)1074 TEST_F(SystemTests, verify_SIGQUIT) {
1075   // Verify that SIGQUIT prints all of the running tests.
1076   Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_job*", "--gtest_also_run_disabled_tests",
1077                                 "-j20"});
1078   // It is expected that all of the tests will be sleeping so nothing will
1079   // complete by the time the signal is sent.
1080   sleep(1);
1081   ASSERT_NE(-1, kill(pid_, SIGQUIT));
1082 
1083   std::vector<char> buffer(4096);
1084   while (true) {
1085     ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1086     if (bytes == -1 && errno == EAGAIN) {
1087       continue;
1088     }
1089     ASSERT_NE(-1, bytes);
1090     if (bytes == 0) {
1091       break;
1092     }
1093     buffer[bytes] = '\0';
1094     raw_output_ += buffer.data();
1095   }
1096   close(fd_);
1097   int status;
1098   ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
1099   SanitizeOutput();
1100   ASSERT_EQ(
1101       "Note: Google Test filter = *.DISABLED_job*\n"
1102       "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1103       "List of current running tests:\n"
1104       "  SystemTests.DISABLED_job_1 (elapsed time XX ms)\n"
1105       "  SystemTests.DISABLED_job_2 (elapsed time XX ms)\n"
1106       "  SystemTests.DISABLED_job_3 (elapsed time XX ms)\n"
1107       "[ RUN      ] SystemTests.DISABLED_job_2\n"
1108       "[       OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
1109       "[ RUN      ] SystemTests.DISABLED_job_3\n"
1110       "[       OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
1111       "[ RUN      ] SystemTests.DISABLED_job_1\n"
1112       "[       OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
1113       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
1114       "[  PASSED  ] 3 tests.\n",
1115       sanitized_output_);
1116   ASSERT_EQ(0, WEXITSTATUS(status));
1117 }
1118 
TEST_F(SystemTests,verify_SIGQUIT_after_test_finish)1119 TEST_F(SystemTests, verify_SIGQUIT_after_test_finish) {
1120   // Verify that SIGQUIT prints all of the tests after a test finishes.
1121   Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_sigquit_*",
1122                                 "--gtest_also_run_disabled_tests", "-j20"});
1123   // It is expected that one tests will have finished, but the rest will still
1124   // be running.
1125   sleep(1);
1126   ASSERT_NE(-1, kill(pid_, SIGQUIT));
1127 
1128   std::vector<char> buffer(4096);
1129   while (true) {
1130     ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1131     if (bytes == -1 && errno == EAGAIN) {
1132       continue;
1133     }
1134     ASSERT_NE(-1, bytes);
1135     if (bytes == 0) {
1136       break;
1137     }
1138     buffer[bytes] = '\0';
1139     raw_output_ += buffer.data();
1140   }
1141   close(fd_);
1142   int status;
1143   ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
1144   SanitizeOutput();
1145   ASSERT_EQ(
1146       "Note: Google Test filter = *.DISABLED_sigquit_*\n"
1147       "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1148       "[ RUN      ] SystemTests.DISABLED_sigquit_no_sleep\n"
1149       "[       OK ] SystemTests.DISABLED_sigquit_no_sleep (XX ms)\n"
1150       "List of current running tests:\n"
1151       "  SystemTests.DISABLED_sigquit_sleep_5 (elapsed time XX ms)\n"
1152       "  SystemTests.DISABLED_sigquit_sleep_6 (elapsed time XX ms)\n"
1153       "[ RUN      ] SystemTests.DISABLED_sigquit_sleep_5\n"
1154       "[       OK ] SystemTests.DISABLED_sigquit_sleep_5 (XX ms)\n"
1155       "[ RUN      ] SystemTests.DISABLED_sigquit_sleep_6\n"
1156       "[       OK ] SystemTests.DISABLED_sigquit_sleep_6 (XX ms)\n"
1157       "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
1158       "[  PASSED  ] 3 tests.\n",
1159       sanitized_output_);
1160   ASSERT_EQ(0, WEXITSTATUS(status));
1161 }
1162 
TEST_F(SystemTests,verify_memory)1163 TEST_F(SystemTests, verify_memory) {
1164   // This test verifies that memory isn't leaking when running repeatedly.
1165   std::vector<const char*> args{"--gtest_filter=*.DISABLED_memory",
1166                                 "--gtest_also_run_disabled_tests", "--gtest_repeat=400"};
1167   ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1168   ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1169   std::vector<std::string> lines(android::base::Split(raw_output_, "\n"));
1170 
1171   constexpr static size_t kMaxLeakBytes = 32 * 1024;
1172   size_t memory_iteration = 0;
1173   size_t memory_start = 0;
1174   size_t memory_last = 0;
1175   for (auto& line : lines) {
1176     size_t memory;
1177     if (android::base::StartsWith(line, "Allocated ") &&
1178         sscanf(line.c_str(), "Allocated %zu", &memory) == 1) {
1179       if (memory_iteration == 0) {
1180         memory_start = memory;
1181       } else {
1182         // Check the increase from the last loop.
1183         if (memory > memory_last) {
1184           ASSERT_GT(kMaxLeakBytes, memory - memory_last)
1185               << "On iteration " << memory_iteration << " memory increased beyond expected value."
1186               << std::endl
1187               << "Last memory bytes " << memory_last << std::endl
1188               << "Current memory bytes " << memory;
1189         }
1190         // Check the increase from the first loop.
1191         if (memory > memory_start) {
1192           ASSERT_GT(kMaxLeakBytes, memory - memory_start)
1193               << "On iteration " << memory_iteration
1194               << " total memory increased beyond expected value." << std::endl
1195               << "Starting memory bytes " << memory_start << std::endl
1196               << "Current memory bytes " << memory;
1197         }
1198       }
1199       memory_last = memory;
1200       memory_iteration++;
1201     }
1202   }
1203   ASSERT_EQ(400, memory_iteration)
1204       << "Did not find the expected 400 lines of memory data." << std::endl
1205       << "Raw output:" << std::endl
1206       << raw_output_;
1207 }
1208 
TEST_F(SystemTests,verify_sharding)1209 TEST_F(SystemTests, verify_sharding) {
1210   std::string expected =
1211       "Output Sorted\n"
1212       "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1213       "Note: This is test shard 1 of 4\n"
1214       "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1215       "[ RUN      ] SystemTestsShard1.DISABLED_case1_test1\n"
1216       "[       OK ] SystemTestsShard1.DISABLED_case1_test1 (XX ms)\n"
1217       "[ RUN      ] SystemTestsShard2.DISABLED_case2_test1\n"
1218       "[       OK ] SystemTestsShard2.DISABLED_case2_test1 (XX ms)\n"
1219       "[ RUN      ] SystemTestsShard3.DISABLED_case3_test1\n"
1220       "[       OK ] SystemTestsShard3.DISABLED_case3_test1 (XX ms)\n"
1221       "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1222       "[  PASSED  ] 3 tests.\n";
1223   ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1224   ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "0", 1));
1225   ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1226 
1227   expected =
1228       "Output Sorted\n"
1229       "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1230       "Note: This is test shard 2 of 4\n"
1231       "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1232       "[ RUN      ] SystemTestsShard1.DISABLED_case1_test2\n"
1233       "[       OK ] SystemTestsShard1.DISABLED_case1_test2 (XX ms)\n"
1234       "[ RUN      ] SystemTestsShard2.DISABLED_case2_test2\n"
1235       "[       OK ] SystemTestsShard2.DISABLED_case2_test2 (XX ms)\n"
1236       "[ RUN      ] SystemTestsShard3.DISABLED_case3_test2\n"
1237       "[       OK ] SystemTestsShard3.DISABLED_case3_test2 (XX ms)\n"
1238       "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1239       "[  PASSED  ] 3 tests.\n";
1240   ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "1", 1));
1241   ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1242 
1243   expected =
1244       "Output Sorted\n"
1245       "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1246       "Note: This is test shard 3 of 4\n"
1247       "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1248       "[ RUN      ] SystemTestsShard1.DISABLED_case1_test3\n"
1249       "[       OK ] SystemTestsShard1.DISABLED_case1_test3 (XX ms)\n"
1250       "[ RUN      ] SystemTestsShard2.DISABLED_case2_test3\n"
1251       "[       OK ] SystemTestsShard2.DISABLED_case2_test3 (XX ms)\n"
1252       "[ RUN      ] SystemTestsShard3.DISABLED_case3_test3\n"
1253       "[       OK ] SystemTestsShard3.DISABLED_case3_test3 (XX ms)\n"
1254       "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1255       "[  PASSED  ] 3 tests.\n";
1256   ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "2", 1));
1257   ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1258 
1259   expected =
1260       "Output Sorted\n"
1261       "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1262       "Note: This is test shard 4 of 4\n"
1263       "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1264       "[ RUN      ] SystemTestsShard1.DISABLED_case1_test4\n"
1265       "[       OK ] SystemTestsShard1.DISABLED_case1_test4 (XX ms)\n"
1266       "[ RUN      ] SystemTestsShard2.DISABLED_case2_test4\n"
1267       "[       OK ] SystemTestsShard2.DISABLED_case2_test4 (XX ms)\n"
1268       "[ RUN      ] SystemTestsShard3.DISABLED_case3_test4\n"
1269       "[       OK ] SystemTestsShard3.DISABLED_case3_test4 (XX ms)\n"
1270       "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1271       "[  PASSED  ] 3 tests.\n";
1272   ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "3", 1));
1273   ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1274 }
1275 
TEST_F(SystemTests,verify_sharding_color)1276 TEST_F(SystemTests, verify_sharding_color) {
1277   std::string expected =
1278       "Output Sorted\n"
1279       "\x1B[0;33mNote: Google Test filter = SystemTestsShard*.DISABLED*\x1B[m\n"
1280       "\x1B[0;33mNote: This is test shard 1 of 4\x1B[m\n"
1281       "\x1B[0;32m[==========]\x1B[m Running 3 tests from 3 test suites (20 jobs).\n"
1282       "\x1B[0;32m[ RUN      ]\x1B[m SystemTestsShard1.DISABLED_case1_test1\n"
1283       "\x1B[0;32m[       OK ]\x1B[m SystemTestsShard1.DISABLED_case1_test1 (XX ms)\n"
1284       "\x1B[0;32m[ RUN      ]\x1B[m SystemTestsShard2.DISABLED_case2_test1\n"
1285       "\x1B[0;32m[       OK ]\x1B[m SystemTestsShard2.DISABLED_case2_test1 (XX ms)\n"
1286       "\x1B[0;32m[ RUN      ]\x1B[m SystemTestsShard3.DISABLED_case3_test1\n"
1287       "\x1B[0;32m[       OK ]\x1B[m SystemTestsShard3.DISABLED_case3_test1 (XX ms)\n"
1288       "\x1B[0;32m[==========]\x1B[m 3 tests from 3 test suites ran. (XX ms total)\n"
1289       "\x1B[0;32m[  PASSED  ]\x1B[m 3 tests.\n";
1290   ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1291   ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "0", 1));
1292   ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0,
1293                                              std::vector<const char*>{"--gtest_color=yes"}));
1294 }
1295 
TEST_F(SystemTests,verify_sharding_error)1296 TEST_F(SystemTests, verify_sharding_error) {
1297   std::string expected =
1298       "Invalid environment variables: we require 0 <= GTEST_SHARD_INDEX < GTEST_TOTAL_SHARDS, but "
1299       "you have GTEST_SHARD_INDEX=4, GTEST_TOTAL_SHARDS=4\n";
1300   ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1301   ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "4", 1));
1302   ASSERT_NO_FATAL_FAILURE(Verify("SystemTestsShard*.DISABLED*", expected, 1));
1303 }
1304 
TEST_F(SystemTests,verify_sharding_error_color)1305 TEST_F(SystemTests, verify_sharding_error_color) {
1306   std::string expected =
1307       "\x1B[0;31mInvalid environment variables: we require 0 <= GTEST_SHARD_INDEX < "
1308       "GTEST_TOTAL_SHARDS, but you have GTEST_SHARD_INDEX=4, GTEST_TOTAL_SHARDS=4\x1B[m\n";
1309   ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1310   ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "4", 1));
1311   ASSERT_NO_FATAL_FAILURE(Verify("SystemTestsShard*.DISABLED*", expected, 1,
1312                                  std::vector<const char*>{"--gtest_color=yes"}));
1313 }
1314 
TEST_F(SystemTests,verify_gtest_flagfile)1315 TEST_F(SystemTests, verify_gtest_flagfile) {
1316   TemporaryFile tf;
1317   ASSERT_TRUE(android::base::WriteStringToFile("--gtest_print_time=0\n", tf.path));
1318   std::string flagfile("--gtest_flagfile=");
1319   flagfile += tf.path;
1320   std::string expected =
1321       "Note: Google Test filter = *.DISABLED_pass\n"
1322       "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1323       "[ RUN      ] SystemTests.DISABLED_pass\n"
1324       "[       OK ] SystemTests.DISABLED_pass\n"
1325       "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1326       "[  PASSED  ] 1 test.\n";
1327   ASSERT_NO_FATAL_FAILURE(
1328       Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{flagfile.c_str()}));
1329 }
1330 
1331 // These tests are used by the verify_disabled tests.
TEST_F(SystemTests,always_pass)1332 TEST_F(SystemTests, always_pass) {}
1333 
TEST_F(SystemTests,DISABLED_always_pass)1334 TEST_F(SystemTests, DISABLED_always_pass) {}
1335 
1336 // The tests listed below will not run by default. They are executed by
1337 // the above tests.
TEST_F(SystemTests,DISABLED_pass)1338 TEST_F(SystemTests, DISABLED_pass) {}
1339 
TEST_F(SystemTests,DISABLED_fail)1340 TEST_F(SystemTests, DISABLED_fail) {
1341   ASSERT_EQ(1, 0);
1342 }
1343 
TEST_F(SystemTests,DISABLED_crash)1344 TEST_F(SystemTests, DISABLED_crash) {
1345   char* p = reinterpret_cast<char*>(static_cast<intptr_t>(atoi("0")));
1346   *p = 3;
1347 }
1348 
TEST_F(SystemTests,DISABLED_sigquit_no_sleep)1349 TEST_F(SystemTests, DISABLED_sigquit_no_sleep) {}
1350 
TEST_F(SystemTests,DISABLED_sigquit_sleep_5)1351 TEST_F(SystemTests, DISABLED_sigquit_sleep_5) {
1352   sleep(5);
1353 }
1354 
TEST_F(SystemTests,DISABLED_sigquit_sleep_6)1355 TEST_F(SystemTests, DISABLED_sigquit_sleep_6) {
1356   sleep(6);
1357 }
1358 
TEST_F(SystemTests,DISABLED_sleep_forever)1359 TEST_F(SystemTests, DISABLED_sleep_forever) {
1360   while (true) {
1361     sleep(10000);
1362   }
1363 }
1364 
TEST_F(SystemTests,DISABLED_sleep5)1365 TEST_F(SystemTests, DISABLED_sleep5) {
1366   sleep(5);
1367 }
1368 
1369 // These tests will finish 1, 2, 3 in non-isolated mode and 3, 2, 1 in isolated
1370 // mode.
TEST_F(SystemTests,DISABLED_order_1)1371 TEST_F(SystemTests, DISABLED_order_1) {
1372   sleep(6);
1373 }
1374 
TEST_F(SystemTests,DISABLED_order_2)1375 TEST_F(SystemTests, DISABLED_order_2) {
1376   sleep(3);
1377 }
1378 
TEST_F(SystemTests,DISABLED_order_3)1379 TEST_F(SystemTests, DISABLED_order_3) {}
1380 
TEST_F(SystemTests,DISABLED_fail_0)1381 TEST_F(SystemTests, DISABLED_fail_0) {
1382   ASSERT_EQ(1, 0);
1383 }
1384 
TEST_F(SystemTests,DISABLED_fail_1)1385 TEST_F(SystemTests, DISABLED_fail_1) {
1386   ASSERT_EQ(1, 0);
1387 }
1388 
TEST_F(SystemTests,DISABLED_fail_2)1389 TEST_F(SystemTests, DISABLED_fail_2) {
1390   ASSERT_EQ(1, 0);
1391 }
1392 
TEST_F(SystemTests,DISABLED_fail_3)1393 TEST_F(SystemTests, DISABLED_fail_3) {
1394   ASSERT_EQ(1, 0);
1395 }
1396 
TEST_F(SystemTests,DISABLED_fail_4)1397 TEST_F(SystemTests, DISABLED_fail_4) {
1398   ASSERT_EQ(1, 0);
1399 }
1400 
TEST_F(SystemTests,DISABLED_fail_5)1401 TEST_F(SystemTests, DISABLED_fail_5) {
1402   ASSERT_EQ(1, 0);
1403 }
1404 
TEST_F(SystemTests,DISABLED_fail_6)1405 TEST_F(SystemTests, DISABLED_fail_6) {
1406   ASSERT_EQ(1, 0);
1407 }
1408 
TEST_F(SystemTests,DISABLED_fail_7)1409 TEST_F(SystemTests, DISABLED_fail_7) {
1410   ASSERT_EQ(1, 0);
1411 }
1412 
TEST_F(SystemTests,DISABLED_fail_8)1413 TEST_F(SystemTests, DISABLED_fail_8) {
1414   ASSERT_EQ(1, 0);
1415 }
1416 
TEST_F(SystemTests,DISABLED_fail_9)1417 TEST_F(SystemTests, DISABLED_fail_9) {
1418   ASSERT_EQ(1, 0);
1419 }
1420 
TEST_F(SystemTests,DISABLED_all_pass_1)1421 TEST_F(SystemTests, DISABLED_all_pass_1) {}
1422 
TEST_F(SystemTests,DISABLED_all_pass_2)1423 TEST_F(SystemTests, DISABLED_all_pass_2) {}
1424 
TEST_F(SystemTests,DISABLED_all_skip_1)1425 TEST_F(SystemTests, DISABLED_all_skip_1) {
1426   GTEST_SKIP();
1427 }
1428 
TEST_F(SystemTests,DISABLED_all_skip_2)1429 TEST_F(SystemTests, DISABLED_all_skip_2) {
1430   GTEST_SKIP() << "Skip message present";
1431 }
1432 
TEST_F(SystemTests,DISABLED_all_slow_1)1433 TEST_F(SystemTests, DISABLED_all_slow_1) {
1434   sleep(3);
1435 }
1436 
TEST_F(SystemTests,DISABLED_all_slow_2)1437 TEST_F(SystemTests, DISABLED_all_slow_2) {
1438   sleep(3);
1439 }
1440 
TEST_F(SystemTests,DISABLED_all_fail_1)1441 TEST_F(SystemTests, DISABLED_all_fail_1) {
1442   ASSERT_EQ(1, 0);
1443 }
1444 
TEST_F(SystemTests,DISABLED_all_fail_2)1445 TEST_F(SystemTests, DISABLED_all_fail_2) {
1446   ASSERT_EQ(1, 0);
1447 }
1448 
TEST_F(SystemTests,DISABLED_all_timeout_1)1449 TEST_F(SystemTests, DISABLED_all_timeout_1) {
1450   sleep(6);
1451 }
1452 
TEST_F(SystemTests,DISABLED_all_timeout_2)1453 TEST_F(SystemTests, DISABLED_all_timeout_2) {
1454   sleep(6);
1455 }
1456 
TEST_F(SystemTests,DISABLED_job_1)1457 TEST_F(SystemTests, DISABLED_job_1) {
1458   sleep(5);
1459 }
1460 
TEST_F(SystemTests,DISABLED_job_2)1461 TEST_F(SystemTests, DISABLED_job_2) {
1462   sleep(3);
1463 }
1464 
TEST_F(SystemTests,DISABLED_job_3)1465 TEST_F(SystemTests, DISABLED_job_3) {
1466   sleep(4);
1467 }
1468 
TEST_F(SystemTests,DISABLED_skip_no_message)1469 TEST_F(SystemTests, DISABLED_skip_no_message) {
1470   GTEST_SKIP();
1471 }
1472 
TEST_F(SystemTests,DISABLED_skip_with_message)1473 TEST_F(SystemTests, DISABLED_skip_with_message) {
1474   GTEST_SKIP() << "This is a skip message";
1475 }
1476 
TEST_F(SystemTests,DISABLED_skip_with_output_before)1477 TEST_F(SystemTests, DISABLED_skip_with_output_before) {
1478   printf("This is the message before the skip message\n");
1479   GTEST_SKIP() << "This is the skip message";
1480 }
1481 
1482 // Do not optimize this call away so that the print after the skip
1483 // will actually occur.
AvoidSkipStopping(int tag=0)1484 void AvoidSkipStopping(int tag = 0) __attribute__((optnone)) {
1485   if (tag == 0) {
1486     GTEST_SKIP() << "This is the skip message";
1487   } else {
1488     GTEST_SKIP() << "This is the skip message " << std::to_string(tag);
1489   }
1490 }
1491 
TEST_F(SystemTests,DISABLED_skip_with_output_after)1492 TEST_F(SystemTests, DISABLED_skip_with_output_after) {
1493   AvoidSkipStopping();
1494   printf("This is the message after the skip message\n");
1495 }
1496 
TEST_F(SystemTests,DISABLED_skip_with_skipped_line)1497 TEST_F(SystemTests, DISABLED_skip_with_skipped_line) {
1498   printf("\nSkipped\n");
1499   AvoidSkipStopping(1);
1500   printf("Skipped\n");
1501   AvoidSkipStopping(2);
1502   printf("Skipped\n");
1503 }
1504 
TEST_F(SystemTests,DISABLED_skip_multiple)1505 TEST_F(SystemTests, DISABLED_skip_multiple) {
1506   printf("This is not a skip message 1\n");
1507   AvoidSkipStopping(1);
1508   printf("This is not a skip message 2\n");
1509   AvoidSkipStopping(2);
1510   AvoidSkipStopping(3);
1511   printf("This is not a skip message 4\n");
1512 }
1513 
1514 class DISABLED_SystemTestsXfail : public ::testing::Test {};
1515 
TEST_F(DISABLED_SystemTestsXfail,xfail_fail)1516 TEST_F(DISABLED_SystemTestsXfail, xfail_fail) {
1517   ASSERT_EQ(1, 0);
1518 }
1519 
TEST_F(DISABLED_SystemTestsXfail,xfail_pass)1520 TEST_F(DISABLED_SystemTestsXfail, xfail_pass) {}
1521 
1522 class SystemTestsDeathTest : public ::testing::Test {
1523  protected:
SetUp()1524   virtual void SetUp() { ::testing::FLAGS_gtest_death_test_style = "threadsafe"; }
1525 };
1526 
DeathTestHelperPass()1527 static void DeathTestHelperPass() {
1528   ASSERT_EQ(1, 1);
1529   exit(0);
1530 }
1531 
TEST_F(SystemTestsDeathTest,DISABLED_death_pass)1532 TEST_F(SystemTestsDeathTest, DISABLED_death_pass) {
1533   ASSERT_EXIT(DeathTestHelperPass(), ::testing::ExitedWithCode(0), "");
1534 }
1535 
DeathTestHelperFail()1536 static void DeathTestHelperFail() {
1537   ASSERT_EQ(1, 0);
1538 }
1539 
TEST_F(SystemTestsDeathTest,DISABLED_death_fail)1540 TEST_F(SystemTestsDeathTest, DISABLED_death_fail) {
1541   ASSERT_EXIT(DeathTestHelperFail(), ::testing::ExitedWithCode(0), "");
1542 }
1543 
TEST(SystemTestsXml1,DISABLED_xml_1)1544 TEST(SystemTestsXml1, DISABLED_xml_1) {}
1545 
TEST(SystemTestsXml1,DISABLED_xml_2)1546 TEST(SystemTestsXml1, DISABLED_xml_2) {
1547   ASSERT_EQ(1, 0);
1548 }
1549 
TEST(SystemTestsXml2,DISABLED_xml_1)1550 TEST(SystemTestsXml2, DISABLED_xml_1) {
1551   ASSERT_EQ(1, 0);
1552 }
1553 
TEST(SystemTestsXml2,DISABLED_xml_2)1554 TEST(SystemTestsXml2, DISABLED_xml_2) {}
1555 
TEST(SystemTestsXml3,DISABLED_xml_1)1556 TEST(SystemTestsXml3, DISABLED_xml_1) {}
1557 
TEST(SystemTestsXml3,DISABLED_xml_2)1558 TEST(SystemTestsXml3, DISABLED_xml_2) {
1559   ASSERT_EQ(1, 0);
1560 }
1561 
TEST(SystemTestsMemory,DISABLED_memory)1562 TEST(SystemTestsMemory, DISABLED_memory) {
1563 #if !defined(__APPLE__)
1564   struct mallinfo info = mallinfo();
1565 #if defined(__ANDROID__)
1566   printf("Allocated %zu\n", info.uordblks);
1567 #else
1568   printf("Allocated %d\n", info.uordblks);
1569 #endif
1570 #else
1571   printf("Allocated 0\n");
1572 #endif
1573 }
1574 
TEST(SystemTestsShard1,DISABLED_case1_test1)1575 TEST(SystemTestsShard1, DISABLED_case1_test1) {}
1576 
TEST(SystemTestsShard1,DISABLED_case1_test2)1577 TEST(SystemTestsShard1, DISABLED_case1_test2) {}
1578 
TEST(SystemTestsShard1,DISABLED_case1_test3)1579 TEST(SystemTestsShard1, DISABLED_case1_test3) {}
1580 
TEST(SystemTestsShard1,DISABLED_case1_test4)1581 TEST(SystemTestsShard1, DISABLED_case1_test4) {}
1582 
TEST(SystemTestsShard2,DISABLED_case2_test1)1583 TEST(SystemTestsShard2, DISABLED_case2_test1) {
1584 }
1585 
TEST(SystemTestsShard2,DISABLED_case2_test2)1586 TEST(SystemTestsShard2, DISABLED_case2_test2) {
1587 }
1588 
TEST(SystemTestsShard2,DISABLED_case2_test3)1589 TEST(SystemTestsShard2, DISABLED_case2_test3) {
1590 }
1591 
TEST(SystemTestsShard2,DISABLED_case2_test4)1592 TEST(SystemTestsShard2, DISABLED_case2_test4) {
1593 }
1594 
TEST(SystemTestsShard3,DISABLED_case3_test1)1595 TEST(SystemTestsShard3, DISABLED_case3_test1) {
1596 }
1597 
TEST(SystemTestsShard3,DISABLED_case3_test2)1598 TEST(SystemTestsShard3, DISABLED_case3_test2) {
1599 }
1600 
TEST(SystemTestsShard3,DISABLED_case3_test3)1601 TEST(SystemTestsShard3, DISABLED_case3_test3) {
1602 }
1603 
TEST(SystemTestsShard3,DISABLED_case3_test4)1604 TEST(SystemTestsShard3, DISABLED_case3_test4) {
1605 }
1606 
1607 }  // namespace gtest_extras
1608 }  // namespace android
1609