1// Copyright 2019 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// This executable runs a series of build commands to test and benchmark some critical user journeys.
16package main
17
18import (
19	"context"
20	"fmt"
21	"os"
22	"path/filepath"
23	"strconv"
24	"strings"
25	"time"
26
27	"android/soong/ui/build"
28	"android/soong/ui/logger"
29	"android/soong/ui/metrics"
30	"android/soong/ui/status"
31	"android/soong/ui/terminal"
32	"android/soong/ui/tracer"
33)
34
35type Test struct {
36	name   string
37	args   []string
38	before func() error
39
40	results TestResults
41}
42
43type TestResults struct {
44	metrics *metrics.Metrics
45	err     error
46}
47
48// Run runs a single build command.  It emulates the "m" command line by calling into Soong UI directly.
49func (t *Test) Run(logsDir string) {
50	output := terminal.NewStatusOutput(os.Stdout, "", false, false)
51
52	log := logger.New(output)
53	defer log.Cleanup()
54
55	ctx, cancel := context.WithCancel(context.Background())
56	defer cancel()
57
58	trace := tracer.New(log)
59	defer trace.Close()
60
61	met := metrics.New()
62
63	stat := &status.Status{}
64	defer stat.Finish()
65	stat.AddOutput(output)
66	stat.AddOutput(trace.StatusTracer())
67
68	build.SetupSignals(log, cancel, func() {
69		trace.Close()
70		log.Cleanup()
71		stat.Finish()
72	})
73
74	buildCtx := build.Context{ContextImpl: &build.ContextImpl{
75		Context: ctx,
76		Logger:  log,
77		Metrics: met,
78		Tracer:  trace,
79		Writer:  output,
80		Status:  stat,
81	}}
82
83	defer logger.Recover(func(err error) {
84		t.results.err = err
85	})
86
87	config := build.NewConfig(buildCtx, t.args...)
88	build.SetupOutDir(buildCtx, config)
89
90	os.MkdirAll(logsDir, 0777)
91	log.SetOutput(filepath.Join(logsDir, "soong.log"))
92	trace.SetOutput(filepath.Join(logsDir, "build.trace"))
93	stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log")))
94	stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log")))
95	stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, "build_error")))
96	stat.AddOutput(status.NewCriticalPath(log))
97
98	defer met.Dump(filepath.Join(logsDir, "soong_metrics"))
99
100	if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
101		if !strings.HasSuffix(start, "N") {
102			if start_time, err := strconv.ParseUint(start, 10, 64); err == nil {
103				log.Verbosef("Took %dms to start up.",
104					time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds())
105				buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano()))
106			}
107		}
108
109		if executable, err := os.Executable(); err == nil {
110			trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
111		}
112	}
113
114	f := build.NewSourceFinder(buildCtx, config)
115	defer f.Shutdown()
116	build.FindSources(buildCtx, config, f)
117
118	build.Build(buildCtx, config, build.BuildAll)
119
120	t.results.metrics = met
121}
122
123// Touch the Intent.java file to cause a rebuild of the frameworks to monitor the
124// incremental build speed as mentioned b/152046247. Intent.java file was chosen
125// as it is a key component of the framework and is often modified.
126func touchIntentFile() error {
127	const intentFileName = "frameworks/base/core/java/android/content/Intent.java"
128	currentTime := time.Now().Local()
129	return os.Chtimes(intentFileName, currentTime, currentTime)
130}
131
132func main() {
133	outDir := os.Getenv("OUT_DIR")
134	if outDir == "" {
135		outDir = "out"
136	}
137
138	cujDir := filepath.Join(outDir, "cuj_tests")
139
140	// Use a subdirectory for the out directory for the tests to keep them isolated.
141	os.Setenv("OUT_DIR", filepath.Join(cujDir, "out"))
142
143	// Each of these tests is run in sequence without resetting the output tree.  The state of the output tree will
144	// affect each successive test.  To maintain the validity of the benchmarks across changes, care must be taken
145	// to avoid changing the state of the tree when a test is run.  This is most easily accomplished by adding tests
146	// at the end.
147	tests := []Test{
148		{
149			// Reset the out directory to get reproducible results.
150			name: "clean",
151			args: []string{"clean"},
152		},
153		{
154			// Parse the build files.
155			name: "nothing",
156			args: []string{"nothing"},
157		},
158		{
159			// Parse the build files again to monitor issues like globs rerunning.
160			name: "nothing_rebuild",
161			args: []string{"nothing"},
162		},
163		{
164			// Parse the build files again, this should always be very short.
165			name: "nothing_rebuild_twice",
166			args: []string{"nothing"},
167		},
168		{
169			// Build the framework as a common developer task and one that keeps getting longer.
170			name: "framework",
171			args: []string{"framework"},
172		},
173		{
174			// Build the framework again to make sure it doesn't rebuild anything.
175			name: "framework_rebuild",
176			args: []string{"framework"},
177		},
178		{
179			// Build the framework again to make sure it doesn't rebuild anything even if it did the second time.
180			name: "framework_rebuild_twice",
181			args: []string{"framework"},
182		},
183		{
184			// Scenario major_inc_build (b/152046247): tracking build speed of major incremental build.
185			name: "major_inc_build_droid",
186			args: []string{"droid"},
187		},
188		{
189			name:   "major_inc_build_framework_minus_apex_after_droid_build",
190			args:   []string{"framework-minus-apex"},
191			before: touchIntentFile,
192		},
193		{
194			name:   "major_inc_build_framework_after_droid_build",
195			args:   []string{"framework"},
196			before: touchIntentFile,
197		},
198		{
199			name:   "major_inc_build_sync_after_droid_build",
200			args:   []string{"sync"},
201			before: touchIntentFile,
202		},
203		{
204			name:   "major_inc_build_droid_rebuild",
205			args:   []string{"droid"},
206			before: touchIntentFile,
207		},
208		{
209			name:   "major_inc_build_update_api_after_droid_rebuild",
210			args:   []string{"update-api"},
211			before: touchIntentFile,
212		},
213	}
214
215	cujMetrics := metrics.NewCriticalUserJourneysMetrics()
216	defer cujMetrics.Dump(filepath.Join(cujDir, "logs", "cuj_metrics.pb"))
217
218	for i, t := range tests {
219		logsSubDir := fmt.Sprintf("%02d_%s", i, t.name)
220		logsDir := filepath.Join(cujDir, "logs", logsSubDir)
221		if t.before != nil {
222			if err := t.before(); err != nil {
223				fmt.Printf("error running before function on test %q: %v\n", t.name, err)
224				break
225			}
226		}
227		t.Run(logsDir)
228		if t.results.err != nil {
229			fmt.Printf("error running test %q: %s\n", t.name, t.results.err)
230			break
231		}
232		if t.results.metrics != nil {
233			cujMetrics.Add(t.name, t.results.metrics)
234		}
235	}
236}
237