commands/run_cvd/main.cc

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <limits.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <fcntl.h>
#include <unistd.h>

#include <algorithm>
#include <functional>
#include <iostream>
#include <fstream>
#include <iomanip>
#include <memory>
#include <sstream>
#include <string>
#include <thread>
#include <vector>

#include <android-base/strings.h>
#include <gflags/gflags.h>
#include <android-base/logging.h>

#include "common/libs/fs/shared_buf.h"
#include "common/libs/fs/shared_fd.h"
#include "common/libs/fs/shared_select.h"
#include "common/libs/utils/environment.h"
#include "common/libs/utils/files.h"
#include "common/libs/utils/network.h"
#include "common/libs/utils/subprocess.h"
#include "common/libs/utils/size_utils.h"
#include "common/libs/utils/tee_logging.h"
#include "host/commands/run_cvd/launch.h"
#include "host/commands/run_cvd/runner_defs.h"
#include "host/commands/run_cvd/process_monitor.h"
#include "host/libs/config/cuttlefish_config.h"
#include "host/libs/config/data_image.h"
#include "host/libs/config/kernel_args.h"
#include "host/commands/kernel_log_monitor/kernel_log_server.h"
#include <host/libs/vm_manager/crosvm_manager.h>
#include "host/libs/vm_manager/vm_manager.h"
#include "host/libs/vm_manager/qemu_manager.h"

using cuttlefish::ForCurrentInstance;
using cuttlefish::RunnerExitCodes;
using cuttlefish::vm_manager::VmManager;

namespace {

cuttlefish::OnSocketReadyCb GetOnSubprocessExitCallback(
    const cuttlefish::CuttlefishConfig& config) {
  if (config.restart_subprocesses()) {
    return cuttlefish::ProcessMonitor::RestartOnExitCb;
  } else {
    return cuttlefish::ProcessMonitor::DoNotMonitorCb;
  }
}

// Maintains the state of the boot process, once a final state is reached
// (success or failure) it sends the appropriate exit code to the foreground
// launcher process
class CvdBootStateMachine {
 public:
  CvdBootStateMachine(cuttlefish::SharedFD fg_launcher_pipe)
      : fg_launcher_pipe_(fg_launcher_pipe), state_(kBootStarted) {}

  // Returns true if the machine is left in a final state
  bool OnBootEvtReceived(cuttlefish::SharedFD boot_events_pipe) {
    monitor::BootEvent evt;
    auto bytes_read = boot_events_pipe->Read(&evt, sizeof(evt));
    if (bytes_read != sizeof(evt)) {
      LOG(ERROR) << "Fail to read a complete event, read " << bytes_read
                 << " bytes only instead of the expected " << sizeof(evt);
      state_ |= kGuestBootFailed;
    } else if (evt == monitor::BootEvent::BootCompleted) {
      LOG(INFO) << "Virtual device booted successfully";
      state_ |= kGuestBootCompleted;
    } else if (evt == monitor::BootEvent::BootFailed) {
      LOG(ERROR) << "Virtual device failed to boot";
      state_ |= kGuestBootFailed;
    }  // Ignore the other signals

    return MaybeWriteToForegroundLauncher();
  }

  bool BootCompleted() const {
    return state_ & kGuestBootCompleted;
  }

  bool BootFailed() const {
    return state_ & kGuestBootFailed;
  }

 private:
  void SendExitCode(cuttlefish::RunnerExitCodes exit_code) {
    fg_launcher_pipe_->Write(&exit_code, sizeof(exit_code));
    // The foreground process will exit after receiving the exit code, if we try
    // to write again we'll get a SIGPIPE
    fg_launcher_pipe_->Close();
  }
  bool MaybeWriteToForegroundLauncher() {
    if (fg_launcher_pipe_->IsOpen()) {
      if (BootCompleted()) {
        SendExitCode(cuttlefish::RunnerExitCodes::kSuccess);
      } else if (state_ & kGuestBootFailed) {
        SendExitCode(cuttlefish::RunnerExitCodes::kVirtualDeviceBootFailed);
      } else {
        // No final state was reached
        return false;
      }
    }
    // Either we sent the code before or just sent it, in any case the state is
    // final
    return true;
  }

  cuttlefish::SharedFD fg_launcher_pipe_;
  int state_;
  static const int kBootStarted = 0;
  static const int kGuestBootCompleted = 1 << 0;
  static const int kGuestBootFailed = 1 << 1;
};

// Abuse the process monitor to make it call us back when boot events are ready
void SetUpHandlingOfBootEvents(
    cuttlefish::ProcessMonitor* process_monitor, cuttlefish::SharedFD boot_events_pipe,
    std::shared_ptr<CvdBootStateMachine> state_machine) {
  process_monitor->MonitorExistingSubprocess(
      // An unused command, so logs are desciptive
      cuttlefish::Command("boot_events_listener"),
      // An unused subprocess, with the boot events pipe as control socket
      cuttlefish::Subprocess(-1, boot_events_pipe),
      [boot_events_pipe, state_machine](cuttlefish::MonitorEntry*) {
        auto sent_code = state_machine->OnBootEvtReceived(boot_events_pipe);
        return !sent_code;
      });
}

bool WriteCuttlefishEnvironment(const cuttlefish::CuttlefishConfig& config) {
  auto env = cuttlefish::SharedFD::Open(config.cuttlefish_env_path().c_str(),
                                 O_CREAT | O_RDWR, 0755);
  if (!env->IsOpen()) {
    LOG(ERROR) << "Unable to create cuttlefish.env file";
    return false;
  }
  auto instance = config.ForDefaultInstance();
  std::string config_env = "export CUTTLEFISH_PER_INSTANCE_PATH=\"" +
                           instance.PerInstancePath(".") + "\"\n";
  config_env += "export ANDROID_SERIAL=" + instance.adb_ip_and_port() + "\n";
  env->Write(config_env.c_str(), config_env.size());
  return true;
}

// Forks and returns the write end of a pipe to the child process. The parent
// process waits for boot events to come through the pipe and exits accordingly.
cuttlefish::SharedFD DaemonizeLauncher(const cuttlefish::CuttlefishConfig& config) {
  auto instance = config.ForDefaultInstance();
  cuttlefish::SharedFD read_end, write_end;
  if (!cuttlefish::SharedFD::Pipe(&read_end, &write_end)) {
    LOG(ERROR) << "Unable to create pipe";
    return cuttlefish::SharedFD(); // a closed FD
  }
  auto pid = fork();
  if (pid) {
    // Explicitly close here, otherwise we may end up reading forever if the
    // child process dies.
    write_end->Close();
    RunnerExitCodes exit_code;
    auto bytes_read = read_end->Read(&exit_code, sizeof(exit_code));
    if (bytes_read != sizeof(exit_code)) {
      LOG(ERROR) << "Failed to read a complete exit code, read " << bytes_read
                 << " bytes only instead of the expected " << sizeof(exit_code);
      exit_code = RunnerExitCodes::kPipeIOError;
    } else if (exit_code == RunnerExitCodes::kSuccess) {
      LOG(INFO) << "Virtual device booted successfully";
    } else if (exit_code == RunnerExitCodes::kVirtualDeviceBootFailed) {
      LOG(ERROR) << "Virtual device failed to boot";
    } else {
      LOG(ERROR) << "Unexpected exit code: " << exit_code;
    }
    if (exit_code == RunnerExitCodes::kSuccess) {
      LOG(INFO) << cuttlefish::kBootCompletedMessage;
    } else {
      LOG(INFO) << cuttlefish::kBootFailedMessage;
    }
    std::exit(exit_code);
  } else {
    // The child returns the write end of the pipe
    if (daemon(/*nochdir*/ 1, /*noclose*/ 1) != 0) {
      LOG(ERROR) << "Failed to daemonize child process: " << strerror(errno);
      std::exit(RunnerExitCodes::kDaemonizationError);
    }
    // Redirect standard I/O
    auto log_path = instance.launcher_log_path();
    auto log =
        cuttlefish::SharedFD::Open(log_path.c_str(), O_CREAT | O_WRONLY | O_APPEND,
                            S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
    if (!log->IsOpen()) {
      LOG(ERROR) << "Failed to create launcher log file: " << log->StrError();
      std::exit(RunnerExitCodes::kDaemonizationError);
    }
    ::android::base::SetLogger(cuttlefish::TeeLogger({
      {cuttlefish::LogFileSeverity(), log},
    }));
    auto dev_null = cuttlefish::SharedFD::Open("/dev/null", O_RDONLY);
    if (!dev_null->IsOpen()) {
      LOG(ERROR) << "Failed to open /dev/null: " << dev_null->StrError();
      std::exit(RunnerExitCodes::kDaemonizationError);
    }
    if (dev_null->UNMANAGED_Dup2(0) < 0) {
      LOG(ERROR) << "Failed dup2 stdin: " << dev_null->StrError();
      std::exit(RunnerExitCodes::kDaemonizationError);
    }
    if (log->UNMANAGED_Dup2(1) < 0) {
      LOG(ERROR) << "Failed dup2 stdout: " << log->StrError();
      std::exit(RunnerExitCodes::kDaemonizationError);
    }
    if (log->UNMANAGED_Dup2(2) < 0) {
      LOG(ERROR) << "Failed dup2 seterr: " << log->StrError();
      std::exit(RunnerExitCodes::kDaemonizationError);
    }

    read_end->Close();
    return write_end;
  }
}

bool CreateQcowOverlay(const std::string& crosvm_path,
                       const std::string& backing_file,
                       const std::string& output_overlay_path) {
  cuttlefish::Command crosvm_qcow2_cmd(crosvm_path);
  crosvm_qcow2_cmd.AddParameter("create_qcow2");
  crosvm_qcow2_cmd.AddParameter("--backing_file=", backing_file);
  crosvm_qcow2_cmd.AddParameter(output_overlay_path);
  int success = crosvm_qcow2_cmd.Start().Wait();
  if (success != 0) {
    LOG(ERROR) << "Unable to run crosvm create_qcow2. Exited with status " << success;
    return false;
  }
  return true;
}

bool PowerwashFiles() {
  auto config = cuttlefish::CuttlefishConfig::Get();
  if (!config) {
    LOG(ERROR) << "Could not load the config.";
    return false;
  }
  using cuttlefish::CreateBlankImage;
  auto instance = config->ForDefaultInstance();

  // TODO(schuffelen): Create these FIFOs in assemble_cvd instead of run_cvd.
  auto kernel_log_pipe = instance.kernel_log_pipe_name();
  unlink(kernel_log_pipe.c_str());

  auto console_in_pipe = instance.console_in_pipe_name();
  unlink(console_in_pipe.c_str());

  auto console_out_pipe = instance.console_out_pipe_name();
  unlink(console_out_pipe.c_str());

  auto logcat_pipe = instance.logcat_pipe_name();
  unlink(logcat_pipe.c_str());

// TODO(schuffelen): Clean up duplication with assemble_cvd
  auto kregistry_path = instance.access_kregistry_path();
  unlink(kregistry_path.c_str());
  CreateBlankImage(kregistry_path, 2 /* mb */, "none");

  auto pstore_path = instance.pstore_path();
  unlink(pstore_path.c_str());
  CreateBlankImage(pstore_path, 2 /* mb */, "none");

  auto sdcard_path = instance.sdcard_path();
  auto sdcard_size = cuttlefish::FileSize(sdcard_path);
  unlink(sdcard_path.c_str());
  // round up
  auto sdcard_mb_size = (sdcard_size + (1 << 20) - 1) / (1 << 20);
  LOG(DEBUG) << "Size in mb is " << sdcard_mb_size;
  CreateBlankImage(sdcard_path, sdcard_mb_size, "sdcard");

  auto overlay_path = instance.PerInstancePath("overlay.img");
  unlink(overlay_path.c_str());
  if (!CreateQcowOverlay(
      config->crosvm_binary(), config->composite_disk_path(), overlay_path)) {
    LOG(ERROR) << "CreateQcowOverlay failed";
    return false;
  }
  return true;
}

void ServerLoop(cuttlefish::SharedFD server,
                cuttlefish::ProcessMonitor* process_monitor) {
  while (true) {
    // TODO: use select to handle simultaneous connections.
    auto client = cuttlefish::SharedFD::Accept(*server);
    cuttlefish::LauncherAction action;
    while (client->IsOpen() && client->Read(&action, sizeof(action)) > 0) {
      switch (action) {
        case cuttlefish::LauncherAction::kStop:
          if (process_monitor->StopMonitoredProcesses()) {
            auto response = cuttlefish::LauncherResponse::kSuccess;
            client->Write(&response, sizeof(response));
            std::exit(0);
          } else {
            auto response = cuttlefish::LauncherResponse::kError;
            client->Write(&response, sizeof(response));
          }
          break;
        case cuttlefish::LauncherAction::kStatus: {
          // TODO(schuffelen): Return more information on a side channel
          auto response = cuttlefish::LauncherResponse::kSuccess;
          client->Write(&response, sizeof(response));
          break;
        }
        case cuttlefish::LauncherAction::kPowerwash: {
          LOG(INFO) << "Received a Powerwash request from the monitor socket";
          if (!process_monitor->StopMonitoredProcesses()) {
            LOG(ERROR) << "Stopping processes failed.";
            auto response = cuttlefish::LauncherResponse::kError;
            client->Write(&response, sizeof(response));
            break;
          }
          if (!PowerwashFiles()) {
            LOG(ERROR) << "Powerwashing files failed.";
            auto response = cuttlefish::LauncherResponse::kError;
            client->Write(&response, sizeof(response));
            break;
          }
          auto response = cuttlefish::LauncherResponse::kSuccess;
          client->Write(&response, sizeof(response));

          auto config = cuttlefish::CuttlefishConfig::Get();
          auto config_path = config->AssemblyPath("cuttlefish_config.json");
          auto followup_stdin =
              cuttlefish::SharedFD::MemfdCreate("pseudo_stdin");
          cuttlefish::WriteAll(followup_stdin, config_path + "\n");
          followup_stdin->LSeek(0, SEEK_SET);
          followup_stdin->UNMANAGED_Dup2(0);

          auto argv_vec = gflags::GetArgvs();
          char** argv = new char*[argv_vec.size() + 1];
          for (size_t i = 0; i < argv_vec.size(); i++) {
            argv[i] = argv_vec[i].data();
          }
          argv[argv_vec.size()] = nullptr;

          execv("/proc/self/exe", argv);
          // execve should not return, so something went wrong.
          PLOG(ERROR) << "execv returned: ";
          response = cuttlefish::LauncherResponse::kError;
          client->Write(&response, sizeof(response));
          break;
        }
        default:
          LOG(ERROR) << "Unrecognized launcher action: "
                     << static_cast<char>(action);
          auto response = cuttlefish::LauncherResponse::kError;
          client->Write(&response, sizeof(response));
      }
    }
  }
}

std::string GetConfigFilePath(const cuttlefish::CuttlefishConfig& config) {
  auto instance = config.ForDefaultInstance();
  return instance.PerInstancePath("cuttlefish_config.json");
}

}  // namespace

int main(int argc, char** argv) {
  setenv("ANDROID_LOG_TAGS", "*:v", /* overwrite */ 0);
  ::android::base::InitLogging(argv, android::base::StderrLogger);
  google::ParseCommandLineFlags(&argc, &argv, false);

  if (isatty(0)) {
    LOG(FATAL) << "stdin was a tty, expected to be passed the output of a previous stage. "
               << "Did you mean to run launch_cvd?";
    return cuttlefish::RunnerExitCodes::kInvalidHostConfiguration;
  } else {
    int error_num = errno;
    if (error_num == EBADF) {
      LOG(FATAL) << "stdin was not a valid file descriptor, expected to be passed the output "
                 << "of assemble_cvd. Did you mean to run launch_cvd?";
      return cuttlefish::RunnerExitCodes::kInvalidHostConfiguration;
    }
  }

  std::string input_files_str;
  {
    auto input_fd = cuttlefish::SharedFD::Dup(0);
    auto bytes_read = cuttlefish::ReadAll(input_fd, &input_files_str);
    if (bytes_read < 0) {
      LOG(FATAL) << "Failed to read input files. Error was \"" << input_fd->StrError() << "\"";
    }
  }
  std::vector<std::string> input_files = android::base::Split(input_files_str, "\n");
  bool found_config = false;
  for (const auto& file : input_files) {
    if (file.find("cuttlefish_config.json") != std::string::npos) {
      found_config = true;
      setenv(cuttlefish::kCuttlefishConfigEnvVarName, file.c_str(), /* overwrite */ false);
    }
  }
  if (!found_config) {
    return RunnerExitCodes::kCuttlefishConfigurationInitError;
  }

  auto config = cuttlefish::CuttlefishConfig::Get();
  auto instance = config->ForDefaultInstance();

  auto log_path = instance.launcher_log_path();

  {
    std::ofstream launcher_log_ofstream(log_path.c_str());
    auto assemble_log = cuttlefish::ReadFile(config->AssemblyPath("assemble_cvd.log"));
    launcher_log_ofstream << assemble_log;
  }
  ::android::base::SetLogger(cuttlefish::LogToStderrAndFiles({log_path}));

  // Change working directory to the instance directory as early as possible to
  // ensure all host processes have the same working dir. This helps stop_cvd
  // find the running processes when it can't establish a communication with the
  // launcher.
  auto chdir_ret = chdir(instance.instance_dir().c_str());
  if (chdir_ret != 0) {
    auto error = errno;
    LOG(ERROR) << "Unable to change dir into instance directory ("
               << instance.instance_dir() << "): " << strerror(error);
    return RunnerExitCodes::kInstanceDirCreationError;
  }

  auto used_tap_devices = cuttlefish::TapInterfacesInUse();
  if (used_tap_devices.count(instance.wifi_tap_name())) {
    LOG(ERROR) << "Wifi TAP device already in use";
    return RunnerExitCodes::kTapDeviceInUse;
  } else if (used_tap_devices.count(instance.mobile_tap_name())) {
    LOG(ERROR) << "Mobile TAP device already in use";
    return RunnerExitCodes::kTapDeviceInUse;
  }

  auto vm_manager = VmManager::Get(config->vm_manager(), config);

  // Check host configuration
  std::vector<std::string> config_commands;
  if (!vm_manager->ValidateHostConfiguration(&config_commands)) {
    LOG(ERROR) << "Validation of user configuration failed";
    std::cout << "Execute the following to correctly configure:" << std::endl;
    for (auto& command : config_commands) {
      std::cout << "  " << command << std::endl;
    }
    std::cout << "You may need to logout for the changes to take effect"
              << std::endl;
    return RunnerExitCodes::kInvalidHostConfiguration;
  }

  if (!WriteCuttlefishEnvironment(*config)) {
    LOG(ERROR) << "Unable to write cuttlefish environment file";
  }

  LOG(INFO) << "The following files contain useful debugging information:";
  if (config->run_as_daemon()) {
    LOG(INFO) << "  Launcher log: " << instance.launcher_log_path();
  }
  LOG(INFO) << "  Android's logcat output: " << instance.logcat_path();
  LOG(INFO) << "  Kernel log: " << instance.PerInstancePath("kernel.log");
  LOG(INFO) << "  Instance configuration: " << GetConfigFilePath(*config);
  LOG(INFO) << "  Instance environment: " << config->cuttlefish_env_path();
  LOG(INFO) << "To access the console run: screen " << instance.console_path();

  auto launcher_monitor_path = instance.launcher_monitor_socket_path();
  auto launcher_monitor_socket = cuttlefish::SharedFD::SocketLocalServer(
      launcher_monitor_path.c_str(), false, SOCK_STREAM, 0666);
  if (!launcher_monitor_socket->IsOpen()) {
    LOG(ERROR) << "Error when opening launcher server: "
               << launcher_monitor_socket->StrError();
    return cuttlefish::RunnerExitCodes::kMonitorCreationFailed;
  }
  cuttlefish::SharedFD foreground_launcher_pipe;
  if (config->run_as_daemon()) {
    foreground_launcher_pipe = DaemonizeLauncher(*config);
    if (!foreground_launcher_pipe->IsOpen()) {
      return RunnerExitCodes::kDaemonizationError;
    }
  } else {
    // Make sure the launcher runs in its own process group even when running in
    // foreground
    if (getsid(0) != getpid()) {
      int retval = setpgid(0, 0);
      if (retval) {
        LOG(ERROR) << "Failed to create new process group: " << strerror(errno);
        std::exit(RunnerExitCodes::kProcessGroupError);
      }
    }
  }

  auto boot_state_machine =
      std::make_shared<CvdBootStateMachine>(foreground_launcher_pipe);

  // Monitor and restart host processes supporting the CVD
  cuttlefish::ProcessMonitor process_monitor;

  if (config->enable_metrics() == cuttlefish::CuttlefishConfig::kYes) {
    LaunchMetrics(&process_monitor, *config);
  }
  LaunchModemSimulatorIfEnabled(*config, &process_monitor);

  auto event_pipes =
      LaunchKernelLogMonitor(*config, &process_monitor, 2);
  cuttlefish::SharedFD boot_events_pipe = event_pipes[0];
  cuttlefish::SharedFD adbd_events_pipe = event_pipes[1];
  event_pipes.clear();

  SetUpHandlingOfBootEvents(&process_monitor, boot_events_pipe,
                            boot_state_machine);

  LaunchLogcatReceiver(*config, &process_monitor);
  LaunchConfigServer(*config, &process_monitor);
  LaunchTombstoneReceiverIfEnabled(*config, &process_monitor);
  LaunchTpm(&process_monitor, *config);
  LaunchSecureEnvironment(&process_monitor, *config);

  // The streamer needs to launch before the VMM because it serves on several
  // sockets (input devices, vsock frame server) when using crosvm.
  StreamerLaunchResult streamer_config;
  if (config->enable_vnc_server()) {
    streamer_config = LaunchVNCServer(
      *config, &process_monitor, GetOnSubprocessExitCallback(*config));
  }
  if (config->enable_webrtc()) {
    streamer_config = LaunchWebRTC(&process_monitor, *config);
  }

  auto kernel_args = KernelCommandLineFromConfig(*config);

  // Start the guest VM
  vm_manager->WithFrontend(streamer_config.launched);
  vm_manager->WithKernelCommandLine(android::base::Join(kernel_args, " "));
  auto vmm_commands = vm_manager->StartCommands();
  for (auto& vmm_cmd: vmm_commands) {
      process_monitor.StartSubprocess(std::move(vmm_cmd),
                                      GetOnSubprocessExitCallback(*config));
  }

  // Start other host processes
  LaunchSocketVsockProxyIfEnabled(&process_monitor, *config);
  LaunchAdbConnectorIfEnabled(&process_monitor, *config, adbd_events_pipe);

  ServerLoop(launcher_monitor_socket, &process_monitor); // Should not return
  LOG(ERROR) << "The server loop returned, it should never happen!!";
  return cuttlefish::RunnerExitCodes::kServerError;
}