1 //
2 // Copyright (C) 2012 The Android Open Source Project
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 //      http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16 
17 #include <netinet/in.h>
18 #include <netinet/ip.h>
19 #include <sys/socket.h>
20 #include <unistd.h>
21 
22 #include <algorithm>
23 #include <memory>
24 #include <string>
25 #include <utility>
26 #include <vector>
27 
28 #include <base/bind.h>
29 #include <base/location.h>
30 #include <base/logging.h>
31 #include <base/message_loop/message_loop.h>
32 #include <base/stl_util.h>
33 #include <base/strings/string_number_conversions.h>
34 #include <base/strings/string_util.h>
35 #include <base/strings/stringprintf.h>
36 #include <base/time/time.h>
37 #include <brillo/message_loops/base_message_loop.h>
38 #include <brillo/message_loops/message_loop.h>
39 #include <brillo/message_loops/message_loop_utils.h>
40 #include <brillo/process.h>
41 #include <brillo/streams/file_stream.h>
42 #include <brillo/streams/stream.h>
43 #include <gtest/gtest.h>
44 
45 #include "update_engine/common/fake_hardware.h"
46 #include "update_engine/common/file_fetcher.h"
47 #include "update_engine/common/http_common.h"
48 #include "update_engine/common/mock_http_fetcher.h"
49 #include "update_engine/common/mock_proxy_resolver.h"
50 #include "update_engine/common/multi_range_http_fetcher.h"
51 #include "update_engine/common/proxy_resolver.h"
52 #include "update_engine/common/test_utils.h"
53 #include "update_engine/common/utils.h"
54 #include "update_engine/libcurl_http_fetcher.h"
55 
56 using brillo::MessageLoop;
57 using std::make_pair;
58 using std::pair;
59 using std::string;
60 using std::unique_ptr;
61 using std::vector;
62 using testing::_;
63 using testing::DoAll;
64 using testing::Return;
65 using testing::SaveArg;
66 
67 namespace {
68 
69 const int kBigLength = 100000;
70 const int kMediumLength = 1000;
71 const int kFlakyTruncateLength = 29000;
72 const int kFlakySleepEvery = 3;
73 const int kFlakySleepSecs = 10;
74 
75 }  // namespace
76 
77 namespace chromeos_update_engine {
78 
79 static const char* kUnusedUrl = "unused://unused";
80 
LocalServerUrlForPath(in_port_t port,const string & path)81 static inline string LocalServerUrlForPath(in_port_t port, const string& path) {
82   string port_str = (port ? base::StringPrintf(":%hu", port) : "");
83   return base::StringPrintf(
84       "http://127.0.0.1%s%s", port_str.c_str(), path.c_str());
85 }
86 
87 //
88 // Class hierarchy for HTTP server implementations.
89 //
90 
91 class HttpServer {
92  public:
93   // This makes it an abstract class (dirty but works).
94   virtual ~HttpServer() = 0;
95 
GetPort() const96   virtual in_port_t GetPort() const { return 0; }
97 
98   bool started_;
99 };
100 
~HttpServer()101 HttpServer::~HttpServer() {}
102 
103 class NullHttpServer : public HttpServer {
104  public:
NullHttpServer()105   NullHttpServer() { started_ = true; }
106 };
107 
108 class PythonHttpServer : public HttpServer {
109  public:
PythonHttpServer()110   PythonHttpServer() : port_(0) {
111     started_ = false;
112 
113     // Spawn the server process.
114     unique_ptr<brillo::Process> http_server(new brillo::ProcessImpl());
115     http_server->AddArg(test_utils::GetBuildArtifactsPath("test_http_server"));
116     http_server->RedirectUsingPipe(STDOUT_FILENO, false);
117 
118     if (!http_server->Start()) {
119       ADD_FAILURE() << "failed to spawn http server process";
120       return;
121     }
122     LOG(INFO) << "started http server with pid " << http_server->pid();
123 
124     // Wait for server to begin accepting connections, obtain its port.
125     brillo::StreamPtr stdout = brillo::FileStream::FromFileDescriptor(
126         http_server->GetPipe(STDOUT_FILENO), false /* own */, nullptr);
127     if (!stdout)
128       return;
129 
130     vector<char> buf(128);
131     string line;
132     while (line.find('\n') == string::npos) {
133       size_t read;
134       if (!stdout->ReadBlocking(buf.data(), buf.size(), &read, nullptr)) {
135         ADD_FAILURE() << "error reading http server stdout";
136         return;
137       }
138       line.append(buf.data(), read);
139       if (read == 0)
140         break;
141     }
142     // Parse the port from the output line.
143     const size_t listening_msg_prefix_len = strlen(kServerListeningMsgPrefix);
144     if (line.size() < listening_msg_prefix_len) {
145       ADD_FAILURE() << "server output too short";
146       return;
147     }
148 
149     EXPECT_EQ(kServerListeningMsgPrefix,
150               line.substr(0, listening_msg_prefix_len));
151     string port_str = line.substr(listening_msg_prefix_len);
152     port_str.resize(port_str.find('\n'));
153     EXPECT_TRUE(base::StringToUint(port_str, &port_));
154 
155     started_ = true;
156     LOG(INFO) << "server running, listening on port " << port_;
157 
158     // Any failure before this point will SIGKILL the test server if started
159     // when the |http_server| goes out of scope.
160     http_server_ = std::move(http_server);
161   }
162 
~PythonHttpServer()163   ~PythonHttpServer() {
164     // If there's no process, do nothing.
165     if (!http_server_)
166       return;
167     // Wait up to 10 seconds for the process to finish. Destroying the process
168     // will kill it with a SIGKILL otherwise.
169     http_server_->Kill(SIGTERM, 10);
170   }
171 
GetPort() const172   in_port_t GetPort() const override { return port_; }
173 
174  private:
175   static const char* kServerListeningMsgPrefix;
176 
177   unique_ptr<brillo::Process> http_server_;
178   unsigned int port_;
179 };
180 
181 const char* PythonHttpServer::kServerListeningMsgPrefix = "listening on port ";
182 
183 //
184 // Class hierarchy for HTTP fetcher test wrappers.
185 //
186 
187 class AnyHttpFetcherTest {
188  public:
AnyHttpFetcherTest()189   AnyHttpFetcherTest() {}
~AnyHttpFetcherTest()190   virtual ~AnyHttpFetcherTest() {}
191 
192   virtual HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) = 0;
NewLargeFetcher(size_t num_proxies)193   HttpFetcher* NewLargeFetcher(size_t num_proxies) {
194     proxy_resolver_.set_num_proxies(num_proxies);
195     return NewLargeFetcher(&proxy_resolver_);
196   }
NewLargeFetcher()197   HttpFetcher* NewLargeFetcher() { return NewLargeFetcher(1); }
198 
199   virtual HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) = 0;
NewSmallFetcher()200   HttpFetcher* NewSmallFetcher() {
201     proxy_resolver_.set_num_proxies(1);
202     return NewSmallFetcher(&proxy_resolver_);
203   }
204 
BigUrl(in_port_t port) const205   virtual string BigUrl(in_port_t port) const { return kUnusedUrl; }
SmallUrl(in_port_t port) const206   virtual string SmallUrl(in_port_t port) const { return kUnusedUrl; }
ErrorUrl(in_port_t port) const207   virtual string ErrorUrl(in_port_t port) const { return kUnusedUrl; }
208 
209   virtual bool IsMock() const = 0;
210   virtual bool IsMulti() const = 0;
211   virtual bool IsHttpSupported() const = 0;
212   virtual bool IsFileFetcher() const = 0;
213 
IgnoreServerAborting(HttpServer * server) const214   virtual void IgnoreServerAborting(HttpServer* server) const {}
215 
216   virtual HttpServer* CreateServer() = 0;
217 
fake_hardware()218   FakeHardware* fake_hardware() { return &fake_hardware_; }
219 
220  protected:
221   DirectProxyResolver proxy_resolver_;
222   FakeHardware fake_hardware_;
223 };
224 
225 class MockHttpFetcherTest : public AnyHttpFetcherTest {
226  public:
227   // Necessary to unhide the definition in the base class.
228   using AnyHttpFetcherTest::NewLargeFetcher;
NewLargeFetcher(ProxyResolver * proxy_resolver)229   HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
230     brillo::Blob big_data(1000000);
231     return new MockHttpFetcher(
232         big_data.data(), big_data.size(), proxy_resolver);
233   }
234 
235   // Necessary to unhide the definition in the base class.
236   using AnyHttpFetcherTest::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)237   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
238     return new MockHttpFetcher("x", 1, proxy_resolver);
239   }
240 
IsMock() const241   bool IsMock() const override { return true; }
IsMulti() const242   bool IsMulti() const override { return false; }
IsHttpSupported() const243   bool IsHttpSupported() const override { return true; }
IsFileFetcher() const244   bool IsFileFetcher() const override { return false; }
245 
CreateServer()246   HttpServer* CreateServer() override { return new NullHttpServer; }
247 };
248 
249 class LibcurlHttpFetcherTest : public AnyHttpFetcherTest {
250  public:
251   // Necessary to unhide the definition in the base class.
252   using AnyHttpFetcherTest::NewLargeFetcher;
NewLargeFetcher(ProxyResolver * proxy_resolver)253   HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
254     LibcurlHttpFetcher* ret =
255         new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_);
256     // Speed up test execution.
257     ret->set_idle_seconds(1);
258     ret->set_retry_seconds(1);
259     fake_hardware_.SetIsOfficialBuild(false);
260     return ret;
261   }
262 
263   // Necessary to unhide the definition in the base class.
264   using AnyHttpFetcherTest::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)265   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
266     return NewLargeFetcher(proxy_resolver);
267   }
268 
BigUrl(in_port_t port) const269   string BigUrl(in_port_t port) const override {
270     return LocalServerUrlForPath(
271         port, base::StringPrintf("/download/%d", kBigLength));
272   }
SmallUrl(in_port_t port) const273   string SmallUrl(in_port_t port) const override {
274     return LocalServerUrlForPath(port, "/foo");
275   }
ErrorUrl(in_port_t port) const276   string ErrorUrl(in_port_t port) const override {
277     return LocalServerUrlForPath(port, "/error");
278   }
279 
IsMock() const280   bool IsMock() const override { return false; }
IsMulti() const281   bool IsMulti() const override { return false; }
IsHttpSupported() const282   bool IsHttpSupported() const override { return true; }
IsFileFetcher() const283   bool IsFileFetcher() const override { return false; }
284 
IgnoreServerAborting(HttpServer * server) const285   void IgnoreServerAborting(HttpServer* server) const override {
286     // Nothing to do.
287   }
288 
CreateServer()289   HttpServer* CreateServer() override { return new PythonHttpServer; }
290 };
291 
292 class MultiRangeHttpFetcherTest : public LibcurlHttpFetcherTest {
293  public:
294   // Necessary to unhide the definition in the base class.
295   using AnyHttpFetcherTest::NewLargeFetcher;
NewLargeFetcher(ProxyResolver * proxy_resolver)296   HttpFetcher* NewLargeFetcher(ProxyResolver* proxy_resolver) override {
297     MultiRangeHttpFetcher* ret = new MultiRangeHttpFetcher(
298         new LibcurlHttpFetcher(proxy_resolver, &fake_hardware_));
299     ret->ClearRanges();
300     ret->AddRange(0);
301     // Speed up test execution.
302     ret->set_idle_seconds(1);
303     ret->set_retry_seconds(1);
304     fake_hardware_.SetIsOfficialBuild(false);
305     return ret;
306   }
307 
308   // Necessary to unhide the definition in the base class.
309   using AnyHttpFetcherTest::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)310   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
311     return NewLargeFetcher(proxy_resolver);
312   }
313 
IsMulti() const314   bool IsMulti() const override { return true; }
315 };
316 
317 class FileFetcherTest : public AnyHttpFetcherTest {
318  public:
319   // Necessary to unhide the definition in the base class.
320   using AnyHttpFetcherTest::NewLargeFetcher;
NewLargeFetcher(ProxyResolver *)321   HttpFetcher* NewLargeFetcher(ProxyResolver* /* proxy_resolver */) override {
322     return new FileFetcher();
323   }
324 
325   // Necessary to unhide the definition in the base class.
326   using AnyHttpFetcherTest::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)327   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
328     return NewLargeFetcher(proxy_resolver);
329   }
330 
BigUrl(in_port_t port) const331   string BigUrl(in_port_t port) const override {
332     static string big_contents = []() {
333       string buf;
334       buf.reserve(kBigLength);
335       constexpr const char* kBigUrlContent = "abcdefghij";
336       for (size_t i = 0; i < kBigLength; i += strlen(kBigUrlContent)) {
337         buf.append(kBigUrlContent,
338                    std::min(kBigLength - i, strlen(kBigUrlContent)));
339       }
340       return buf;
341     }();
342     test_utils::WriteFileString(temp_file_.path(), big_contents);
343     return "file://" + temp_file_.path();
344   }
SmallUrl(in_port_t port) const345   string SmallUrl(in_port_t port) const override {
346     test_utils::WriteFileString(temp_file_.path(), "small contents");
347     return "file://" + temp_file_.path();
348   }
ErrorUrl(in_port_t port) const349   string ErrorUrl(in_port_t port) const override {
350     return "file:///path/to/non-existing-file";
351   }
352 
IsMock() const353   bool IsMock() const override { return false; }
IsMulti() const354   bool IsMulti() const override { return false; }
IsHttpSupported() const355   bool IsHttpSupported() const override { return false; }
IsFileFetcher() const356   bool IsFileFetcher() const override { return true; }
357 
IgnoreServerAborting(HttpServer * server) const358   void IgnoreServerAborting(HttpServer* server) const override {}
359 
CreateServer()360   HttpServer* CreateServer() override { return new NullHttpServer; }
361 
362  private:
363   test_utils::ScopedTempFile temp_file_{"ue_file_fetcher.XXXXXX"};
364 };
365 
366 class MultiRangeHttpFetcherOverFileFetcherTest : public FileFetcherTest {
367  public:
368   // Necessary to unhide the definition in the base class.
369   using AnyHttpFetcherTest::NewLargeFetcher;
NewLargeFetcher(ProxyResolver *)370   HttpFetcher* NewLargeFetcher(ProxyResolver* /* proxy_resolver */) override {
371     MultiRangeHttpFetcher* ret = new MultiRangeHttpFetcher(new FileFetcher());
372     ret->ClearRanges();
373     // FileFetcher doesn't support range with unspecified length.
374     ret->AddRange(0, 1);
375     // Speed up test execution.
376     ret->set_idle_seconds(1);
377     ret->set_retry_seconds(1);
378     fake_hardware_.SetIsOfficialBuild(false);
379     return ret;
380   }
381 
382   // Necessary to unhide the definition in the base class.
383   using AnyHttpFetcherTest::NewSmallFetcher;
NewSmallFetcher(ProxyResolver * proxy_resolver)384   HttpFetcher* NewSmallFetcher(ProxyResolver* proxy_resolver) override {
385     return NewLargeFetcher(proxy_resolver);
386   }
387 
IsMulti() const388   bool IsMulti() const override { return true; }
389 };
390 
391 //
392 // Infrastructure for type tests of HTTP fetcher.
393 // See: http://code.google.com/p/googletest/wiki/AdvancedGuide#Typed_Tests
394 //
395 
396 // Fixture class template. We use an explicit constraint to guarantee that it
397 // can only be instantiated with an AnyHttpFetcherTest type, see:
398 // http://www2.research.att.com/~bs/bs_faq2.html#constraints
399 template <typename T>
400 class HttpFetcherTest : public ::testing::Test {
401  public:
402   base::MessageLoopForIO base_loop_;
403   brillo::BaseMessageLoop loop_{&base_loop_};
404 
405   T test_;
406 
407  protected:
HttpFetcherTest()408   HttpFetcherTest() { loop_.SetAsCurrent(); }
409 
TearDown()410   void TearDown() override {
411     EXPECT_EQ(0, brillo::MessageLoopRunMaxIterations(&loop_, 1));
412   }
413 
414  private:
TypeConstraint(T * a)415   static void TypeConstraint(T* a) {
416     AnyHttpFetcherTest* b = a;
417     if (b == 0)  // Silence compiler warning of unused variable.
418       *b = a;
419   }
420 };
421 
422 // Test case types list.
423 typedef ::testing::Types<LibcurlHttpFetcherTest,
424                          MockHttpFetcherTest,
425                          MultiRangeHttpFetcherTest,
426                          FileFetcherTest,
427                          MultiRangeHttpFetcherOverFileFetcherTest>
428     HttpFetcherTestTypes;
429 TYPED_TEST_CASE(HttpFetcherTest, HttpFetcherTestTypes);
430 
431 namespace {
432 class HttpFetcherTestDelegate : public HttpFetcherDelegate {
433  public:
434   HttpFetcherTestDelegate() = default;
435 
ReceivedBytes(HttpFetcher *,const void * bytes,size_t length)436   bool ReceivedBytes(HttpFetcher* /* fetcher */,
437                      const void* bytes,
438                      size_t length) override {
439     data.append(reinterpret_cast<const char*>(bytes), length);
440     // Update counters
441     times_received_bytes_called_++;
442     return true;
443   }
444 
TransferComplete(HttpFetcher * fetcher,bool successful)445   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
446     if (is_expect_error_)
447       EXPECT_EQ(kHttpResponseNotFound, fetcher->http_response_code());
448     else
449       EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
450     MessageLoop::current()->BreakLoop();
451 
452     // Update counter
453     times_transfer_complete_called_++;
454   }
455 
TransferTerminated(HttpFetcher * fetcher)456   void TransferTerminated(HttpFetcher* fetcher) override {
457     times_transfer_terminated_called_++;
458     MessageLoop::current()->BreakLoop();
459   }
460 
461   // Are we expecting an error response? (default: no)
462   bool is_expect_error_{false};
463 
464   // Counters for callback invocations.
465   int times_transfer_complete_called_{0};
466   int times_transfer_terminated_called_{0};
467   int times_received_bytes_called_{0};
468 
469   // The received data bytes.
470   string data;
471 };
472 
StartTransfer(HttpFetcher * http_fetcher,const string & url)473 void StartTransfer(HttpFetcher* http_fetcher, const string& url) {
474   http_fetcher->BeginTransfer(url);
475 }
476 }  // namespace
477 
TYPED_TEST(HttpFetcherTest,SimpleTest)478 TYPED_TEST(HttpFetcherTest, SimpleTest) {
479   HttpFetcherTestDelegate delegate;
480   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
481   fetcher->set_delegate(&delegate);
482 
483   unique_ptr<HttpServer> server(this->test_.CreateServer());
484   ASSERT_TRUE(server->started_);
485 
486   this->loop_.PostTask(FROM_HERE,
487                        base::Bind(StartTransfer,
488                                   fetcher.get(),
489                                   this->test_.SmallUrl(server->GetPort())));
490   this->loop_.Run();
491   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
492 }
493 
TYPED_TEST(HttpFetcherTest,SimpleBigTest)494 TYPED_TEST(HttpFetcherTest, SimpleBigTest) {
495   HttpFetcherTestDelegate delegate;
496   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
497   fetcher->set_delegate(&delegate);
498 
499   unique_ptr<HttpServer> server(this->test_.CreateServer());
500   ASSERT_TRUE(server->started_);
501 
502   this->loop_.PostTask(
503       FROM_HERE,
504       base::Bind(
505           StartTransfer, fetcher.get(), this->test_.BigUrl(server->GetPort())));
506   this->loop_.Run();
507   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
508 }
509 
510 // Issue #9648: when server returns an error HTTP response, the fetcher needs to
511 // terminate transfer prematurely, rather than try to process the error payload.
TYPED_TEST(HttpFetcherTest,ErrorTest)512 TYPED_TEST(HttpFetcherTest, ErrorTest) {
513   if (this->test_.IsMock() || this->test_.IsMulti())
514     return;
515   HttpFetcherTestDelegate delegate;
516 
517   // Delegate should expect an error response.
518   delegate.is_expect_error_ = true;
519 
520   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
521   fetcher->set_delegate(&delegate);
522 
523   unique_ptr<HttpServer> server(this->test_.CreateServer());
524   ASSERT_TRUE(server->started_);
525 
526   this->loop_.PostTask(FROM_HERE,
527                        base::Bind(StartTransfer,
528                                   fetcher.get(),
529                                   this->test_.ErrorUrl(server->GetPort())));
530   this->loop_.Run();
531 
532   // Make sure that no bytes were received.
533   EXPECT_EQ(0, delegate.times_received_bytes_called_);
534   EXPECT_EQ(0U, fetcher->GetBytesDownloaded());
535 
536   // Make sure that transfer completion was signaled once, and no termination
537   // was signaled.
538   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
539   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
540 }
541 
TYPED_TEST(HttpFetcherTest,ExtraHeadersInRequestTest)542 TYPED_TEST(HttpFetcherTest, ExtraHeadersInRequestTest) {
543   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
544     return;
545 
546   HttpFetcherTestDelegate delegate;
547   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
548   fetcher->set_delegate(&delegate);
549   fetcher->SetHeader("User-Agent", "MyTest");
550   fetcher->SetHeader("user-agent", "Override that header");
551   fetcher->SetHeader("Authorization", "Basic user:passwd");
552 
553   // Invalid headers.
554   fetcher->SetHeader("X-Foo", "Invalid\nHeader\nIgnored");
555   fetcher->SetHeader("X-Bar: ", "I do not know how to parse");
556 
557   // Hide Accept header normally added by default.
558   fetcher->SetHeader("Accept", "");
559 
560   PythonHttpServer server;
561   int port = server.GetPort();
562   ASSERT_TRUE(server.started_);
563 
564   this->loop_.PostTask(
565       FROM_HERE,
566       base::Bind(StartTransfer,
567                  fetcher.get(),
568                  LocalServerUrlForPath(port, "/echo-headers")));
569   this->loop_.Run();
570 
571   EXPECT_NE(string::npos,
572             delegate.data.find("user-agent: Override that header\r\n"));
573   EXPECT_NE(string::npos,
574             delegate.data.find("Authorization: Basic user:passwd\r\n"));
575 
576   EXPECT_EQ(string::npos, delegate.data.find("\nAccept:"));
577   EXPECT_EQ(string::npos, delegate.data.find("X-Foo: Invalid"));
578   EXPECT_EQ(string::npos, delegate.data.find("X-Bar: I do not"));
579 }
580 
581 namespace {
582 class PausingHttpFetcherTestDelegate : public HttpFetcherDelegate {
583  public:
ReceivedBytes(HttpFetcher * fetcher,const void *,size_t)584   bool ReceivedBytes(HttpFetcher* fetcher,
585                      const void* /* bytes */,
586                      size_t /* length */) override {
587     CHECK(!paused_);
588     paused_ = true;
589     fetcher->Pause();
590     return true;
591   }
TransferComplete(HttpFetcher * fetcher,bool successful)592   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
593     MessageLoop::current()->BreakLoop();
594   }
TransferTerminated(HttpFetcher * fetcher)595   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
Unpause()596   void Unpause() {
597     CHECK(paused_);
598     paused_ = false;
599     fetcher_->Unpause();
600   }
601   bool paused_;
602   HttpFetcher* fetcher_;
603 };
604 
UnpausingTimeoutCallback(PausingHttpFetcherTestDelegate * delegate,MessageLoop::TaskId * my_id)605 void UnpausingTimeoutCallback(PausingHttpFetcherTestDelegate* delegate,
606                               MessageLoop::TaskId* my_id) {
607   if (delegate->paused_)
608     delegate->Unpause();
609   // Update the task id with the new scheduled callback.
610   *my_id = MessageLoop::current()->PostDelayedTask(
611       FROM_HERE,
612       base::Bind(&UnpausingTimeoutCallback, delegate, my_id),
613       base::TimeDelta::FromMilliseconds(200));
614 }
615 }  // namespace
616 
TYPED_TEST(HttpFetcherTest,PauseTest)617 TYPED_TEST(HttpFetcherTest, PauseTest) {
618   PausingHttpFetcherTestDelegate delegate;
619   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher());
620   delegate.paused_ = false;
621   delegate.fetcher_ = fetcher.get();
622   fetcher->set_delegate(&delegate);
623 
624   unique_ptr<HttpServer> server(this->test_.CreateServer());
625   ASSERT_TRUE(server->started_);
626 
627   MessageLoop::TaskId callback_id;
628   callback_id = this->loop_.PostDelayedTask(
629       FROM_HERE,
630       base::Bind(&UnpausingTimeoutCallback, &delegate, &callback_id),
631       base::TimeDelta::FromMilliseconds(200));
632   fetcher->BeginTransfer(this->test_.BigUrl(server->GetPort()));
633 
634   this->loop_.Run();
635   EXPECT_TRUE(this->loop_.CancelTask(callback_id));
636 }
637 
638 // This test will pause the fetcher while the download is not yet started
639 // because it is waiting for the proxy to be resolved.
TYPED_TEST(HttpFetcherTest,PauseWhileResolvingProxyTest)640 TYPED_TEST(HttpFetcherTest, PauseWhileResolvingProxyTest) {
641   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
642     return;
643   MockProxyResolver mock_resolver;
644   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher(&mock_resolver));
645 
646   // Saved arguments from the proxy call.
647   ProxiesResolvedFn proxy_callback;
648   EXPECT_CALL(mock_resolver, GetProxiesForUrl("http://fake_url", _))
649       .WillOnce(DoAll(SaveArg<1>(&proxy_callback), Return(true)));
650   fetcher->BeginTransfer("http://fake_url");
651   testing::Mock::VerifyAndClearExpectations(&mock_resolver);
652 
653   // Pausing and unpausing while resolving the proxy should not affect anything.
654   fetcher->Pause();
655   fetcher->Unpause();
656   fetcher->Pause();
657   // Proxy resolver comes back after we paused the fetcher.
658   ASSERT_FALSE(proxy_callback.is_null());
659   proxy_callback.Run({1, kNoProxy});
660 }
661 
662 namespace {
663 class AbortingHttpFetcherTestDelegate : public HttpFetcherDelegate {
664  public:
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)665   bool ReceivedBytes(HttpFetcher* fetcher,
666                      const void* bytes,
667                      size_t length) override {
668     return true;
669   }
TransferComplete(HttpFetcher * fetcher,bool successful)670   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
671     ADD_FAILURE();  // We should never get here
672     MessageLoop::current()->BreakLoop();
673   }
TransferTerminated(HttpFetcher * fetcher)674   void TransferTerminated(HttpFetcher* fetcher) override {
675     EXPECT_EQ(fetcher, fetcher_.get());
676     EXPECT_FALSE(once_);
677     EXPECT_TRUE(callback_once_);
678     callback_once_ = false;
679     // The fetcher could have a callback scheduled on the ProxyResolver that
680     // can fire after this callback. We wait until the end of the test to
681     // delete the fetcher.
682   }
TerminateTransfer()683   void TerminateTransfer() {
684     CHECK(once_);
685     once_ = false;
686     fetcher_->TerminateTransfer();
687   }
EndLoop()688   void EndLoop() { MessageLoop::current()->BreakLoop(); }
689   bool once_;
690   bool callback_once_;
691   unique_ptr<HttpFetcher> fetcher_;
692 };
693 
AbortingTimeoutCallback(AbortingHttpFetcherTestDelegate * delegate,MessageLoop::TaskId * my_id)694 void AbortingTimeoutCallback(AbortingHttpFetcherTestDelegate* delegate,
695                              MessageLoop::TaskId* my_id) {
696   if (delegate->once_) {
697     delegate->TerminateTransfer();
698     *my_id = MessageLoop::current()->PostTask(
699         FROM_HERE, base::Bind(AbortingTimeoutCallback, delegate, my_id));
700   } else {
701     delegate->EndLoop();
702     *my_id = MessageLoop::kTaskIdNull;
703   }
704 }
705 }  // namespace
706 
TYPED_TEST(HttpFetcherTest,AbortTest)707 TYPED_TEST(HttpFetcherTest, AbortTest) {
708   AbortingHttpFetcherTestDelegate delegate;
709   delegate.fetcher_.reset(this->test_.NewLargeFetcher());
710   delegate.once_ = true;
711   delegate.callback_once_ = true;
712   delegate.fetcher_->set_delegate(&delegate);
713 
714   unique_ptr<HttpServer> server(this->test_.CreateServer());
715   this->test_.IgnoreServerAborting(server.get());
716   ASSERT_TRUE(server->started_);
717 
718   MessageLoop::TaskId task_id = MessageLoop::kTaskIdNull;
719 
720   task_id = this->loop_.PostTask(
721       FROM_HERE, base::Bind(AbortingTimeoutCallback, &delegate, &task_id));
722   delegate.fetcher_->BeginTransfer(this->test_.BigUrl(server->GetPort()));
723 
724   this->loop_.Run();
725   CHECK(!delegate.once_);
726   CHECK(!delegate.callback_once_);
727   this->loop_.CancelTask(task_id);
728 }
729 
TYPED_TEST(HttpFetcherTest,TerminateTransferWhileResolvingProxyTest)730 TYPED_TEST(HttpFetcherTest, TerminateTransferWhileResolvingProxyTest) {
731   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
732     return;
733   MockProxyResolver mock_resolver;
734   unique_ptr<HttpFetcher> fetcher(this->test_.NewLargeFetcher(&mock_resolver));
735 
736   HttpFetcherTestDelegate delegate;
737   fetcher->set_delegate(&delegate);
738 
739   EXPECT_CALL(mock_resolver, GetProxiesForUrl(_, _)).WillOnce(Return(123));
740   fetcher->BeginTransfer("http://fake_url");
741   // Run the message loop until idle. This must call the MockProxyResolver with
742   // the request.
743   while (this->loop_.RunOnce(false)) {
744   }
745   testing::Mock::VerifyAndClearExpectations(&mock_resolver);
746 
747   EXPECT_CALL(mock_resolver, CancelProxyRequest(123)).WillOnce(Return(true));
748 
749   // Terminate the transfer right before the proxy resolution response.
750   fetcher->TerminateTransfer();
751   EXPECT_EQ(0, delegate.times_received_bytes_called_);
752   EXPECT_EQ(0, delegate.times_transfer_complete_called_);
753   EXPECT_EQ(1, delegate.times_transfer_terminated_called_);
754 }
755 
756 namespace {
757 class FlakyHttpFetcherTestDelegate : public HttpFetcherDelegate {
758  public:
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)759   bool ReceivedBytes(HttpFetcher* fetcher,
760                      const void* bytes,
761                      size_t length) override {
762     data.append(reinterpret_cast<const char*>(bytes), length);
763     return true;
764   }
TransferComplete(HttpFetcher * fetcher,bool successful)765   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
766     EXPECT_TRUE(successful);
767     EXPECT_EQ(kHttpResponsePartialContent, fetcher->http_response_code());
768     MessageLoop::current()->BreakLoop();
769   }
TransferTerminated(HttpFetcher * fetcher)770   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
771   string data;
772 };
773 }  // namespace
774 
TYPED_TEST(HttpFetcherTest,FlakyTest)775 TYPED_TEST(HttpFetcherTest, FlakyTest) {
776   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
777     return;
778   {
779     FlakyHttpFetcherTestDelegate delegate;
780     unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
781     fetcher->set_delegate(&delegate);
782 
783     unique_ptr<HttpServer> server(this->test_.CreateServer());
784     ASSERT_TRUE(server->started_);
785 
786     this->loop_.PostTask(FROM_HERE,
787                          base::Bind(&StartTransfer,
788                                     fetcher.get(),
789                                     LocalServerUrlForPath(
790                                         server->GetPort(),
791                                         base::StringPrintf("/flaky/%d/%d/%d/%d",
792                                                            kBigLength,
793                                                            kFlakyTruncateLength,
794                                                            kFlakySleepEvery,
795                                                            kFlakySleepSecs))));
796     this->loop_.Run();
797 
798     // verify the data we get back
799     ASSERT_EQ(kBigLength, static_cast<int>(delegate.data.size()));
800     for (int i = 0; i < kBigLength; i += 10) {
801       // Assert so that we don't flood the screen w/ EXPECT errors on failure.
802       ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
803     }
804   }
805 }
806 
807 namespace {
808 // This delegate kills the server attached to it after receiving any bytes.
809 // This can be used for testing what happens when you try to fetch data and
810 // the server dies.
811 class FailureHttpFetcherTestDelegate : public HttpFetcherDelegate {
812  public:
FailureHttpFetcherTestDelegate(PythonHttpServer * server)813   explicit FailureHttpFetcherTestDelegate(PythonHttpServer* server)
814       : server_(server) {}
815 
~FailureHttpFetcherTestDelegate()816   ~FailureHttpFetcherTestDelegate() override {
817     if (server_) {
818       LOG(INFO) << "Stopping server in destructor";
819       server_.reset();
820       LOG(INFO) << "server stopped";
821     }
822   }
823 
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)824   bool ReceivedBytes(HttpFetcher* fetcher,
825                      const void* bytes,
826                      size_t length) override {
827     if (server_) {
828       LOG(INFO) << "Stopping server in ReceivedBytes";
829       server_.reset();
830       LOG(INFO) << "server stopped";
831     }
832     return true;
833   }
TransferComplete(HttpFetcher * fetcher,bool successful)834   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
835     EXPECT_FALSE(successful);
836     EXPECT_EQ(0, fetcher->http_response_code());
837     times_transfer_complete_called_++;
838     MessageLoop::current()->BreakLoop();
839   }
TransferTerminated(HttpFetcher * fetcher)840   void TransferTerminated(HttpFetcher* fetcher) override {
841     times_transfer_terminated_called_++;
842     MessageLoop::current()->BreakLoop();
843   }
844   unique_ptr<PythonHttpServer> server_;
845   int times_transfer_terminated_called_{0};
846   int times_transfer_complete_called_{0};
847 };
848 }  // namespace
849 
TYPED_TEST(HttpFetcherTest,FailureTest)850 TYPED_TEST(HttpFetcherTest, FailureTest) {
851   // This test ensures that a fetcher responds correctly when a server isn't
852   // available at all.
853   if (this->test_.IsMock())
854     return;
855   FailureHttpFetcherTestDelegate delegate(nullptr);
856   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
857   fetcher->set_delegate(&delegate);
858 
859   this->loop_.PostTask(
860       FROM_HERE,
861       base::Bind(
862           StartTransfer, fetcher.get(), "http://host_doesnt_exist99999999"));
863   this->loop_.Run();
864   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
865   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
866 
867   // Exiting and testing happens in the delegate
868 }
869 
TYPED_TEST(HttpFetcherTest,NoResponseTest)870 TYPED_TEST(HttpFetcherTest, NoResponseTest) {
871   // This test starts a new http server but the server doesn't respond and just
872   // closes the connection.
873   if (this->test_.IsMock())
874     return;
875 
876   PythonHttpServer* server = new PythonHttpServer();
877   int port = server->GetPort();
878   ASSERT_TRUE(server->started_);
879 
880   // Handles destruction and claims ownership.
881   FailureHttpFetcherTestDelegate delegate(server);
882   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
883   fetcher->set_delegate(&delegate);
884   // The server will not reply at all, so we can limit the execution time of the
885   // test by reducing the low-speed timeout to something small. The test will
886   // finish once the TimeoutCallback() triggers (every second) and the timeout
887   // expired.
888   fetcher->set_low_speed_limit(kDownloadLowSpeedLimitBps, 1);
889 
890   this->loop_.PostTask(
891       FROM_HERE,
892       base::Bind(
893           StartTransfer, fetcher.get(), LocalServerUrlForPath(port, "/hang")));
894   this->loop_.Run();
895   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
896   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
897 
898   // Check that no other callback runs in the next two seconds. That would
899   // indicate a leaked callback.
900   bool timeout = false;
901   auto callback = base::Bind([](bool* timeout) { *timeout = true; },
902                              base::Unretained(&timeout));
903   this->loop_.PostDelayedTask(
904       FROM_HERE, callback, base::TimeDelta::FromSeconds(2));
905   EXPECT_TRUE(this->loop_.RunOnce(true));
906   EXPECT_TRUE(timeout);
907 }
908 
TYPED_TEST(HttpFetcherTest,ServerDiesTest)909 TYPED_TEST(HttpFetcherTest, ServerDiesTest) {
910   // This test starts a new http server and kills it after receiving its first
911   // set of bytes. It test whether or not our fetcher eventually gives up on
912   // retries and aborts correctly.
913   if (this->test_.IsMock())
914     return;
915   PythonHttpServer* server = new PythonHttpServer();
916   int port = server->GetPort();
917   ASSERT_TRUE(server->started_);
918 
919   // Handles destruction and claims ownership.
920   FailureHttpFetcherTestDelegate delegate(server);
921   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
922   fetcher->set_delegate(&delegate);
923 
924   this->loop_.PostTask(
925       FROM_HERE,
926       base::Bind(StartTransfer,
927                  fetcher.get(),
928                  LocalServerUrlForPath(port,
929                                        base::StringPrintf("/flaky/%d/%d/%d/%d",
930                                                           kBigLength,
931                                                           kFlakyTruncateLength,
932                                                           kFlakySleepEvery,
933                                                           kFlakySleepSecs))));
934   this->loop_.Run();
935   EXPECT_EQ(1, delegate.times_transfer_complete_called_);
936   EXPECT_EQ(0, delegate.times_transfer_terminated_called_);
937 
938   // Exiting and testing happens in the delegate
939 }
940 
941 // Test that we can cancel a transfer while it is still trying to connect to the
942 // server. This test kills the server after a few bytes are received.
TYPED_TEST(HttpFetcherTest,TerminateTransferWhenServerDiedTest)943 TYPED_TEST(HttpFetcherTest, TerminateTransferWhenServerDiedTest) {
944   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
945     return;
946 
947   PythonHttpServer* server = new PythonHttpServer();
948   int port = server->GetPort();
949   ASSERT_TRUE(server->started_);
950 
951   // Handles destruction and claims ownership.
952   FailureHttpFetcherTestDelegate delegate(server);
953   unique_ptr<HttpFetcher> fetcher(this->test_.NewSmallFetcher());
954   fetcher->set_delegate(&delegate);
955 
956   this->loop_.PostTask(
957       FROM_HERE,
958       base::Bind(StartTransfer,
959                  fetcher.get(),
960                  LocalServerUrlForPath(port,
961                                        base::StringPrintf("/flaky/%d/%d/%d/%d",
962                                                           kBigLength,
963                                                           kFlakyTruncateLength,
964                                                           kFlakySleepEvery,
965                                                           kFlakySleepSecs))));
966   // Terminating the transfer after 3 seconds gives it a chance to contact the
967   // server and enter the retry loop.
968   this->loop_.PostDelayedTask(FROM_HERE,
969                               base::Bind(&HttpFetcher::TerminateTransfer,
970                                          base::Unretained(fetcher.get())),
971                               base::TimeDelta::FromSeconds(3));
972 
973   // Exiting and testing happens in the delegate.
974   this->loop_.Run();
975   EXPECT_EQ(0, delegate.times_transfer_complete_called_);
976   EXPECT_EQ(1, delegate.times_transfer_terminated_called_);
977 
978   // Check that no other callback runs in the next two seconds. That would
979   // indicate a leaked callback.
980   bool timeout = false;
981   auto callback = base::Bind([](bool* timeout) { *timeout = true; },
982                              base::Unretained(&timeout));
983   this->loop_.PostDelayedTask(
984       FROM_HERE, callback, base::TimeDelta::FromSeconds(2));
985   EXPECT_TRUE(this->loop_.RunOnce(true));
986   EXPECT_TRUE(timeout);
987 }
988 
989 namespace {
990 const HttpResponseCode kRedirectCodes[] = {kHttpResponseMovedPermanently,
991                                            kHttpResponseFound,
992                                            kHttpResponseSeeOther,
993                                            kHttpResponseTempRedirect};
994 
995 class RedirectHttpFetcherTestDelegate : public HttpFetcherDelegate {
996  public:
RedirectHttpFetcherTestDelegate(bool expected_successful)997   explicit RedirectHttpFetcherTestDelegate(bool expected_successful)
998       : expected_successful_(expected_successful) {}
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)999   bool ReceivedBytes(HttpFetcher* fetcher,
1000                      const void* bytes,
1001                      size_t length) override {
1002     data.append(reinterpret_cast<const char*>(bytes), length);
1003     return true;
1004   }
TransferComplete(HttpFetcher * fetcher,bool successful)1005   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1006     EXPECT_EQ(expected_successful_, successful);
1007     if (expected_successful_) {
1008       EXPECT_EQ(kHttpResponseOk, fetcher->http_response_code());
1009     } else {
1010       EXPECT_GE(fetcher->http_response_code(), kHttpResponseMovedPermanently);
1011       EXPECT_LE(fetcher->http_response_code(), kHttpResponseTempRedirect);
1012     }
1013     MessageLoop::current()->BreakLoop();
1014   }
TransferTerminated(HttpFetcher * fetcher)1015   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
1016   bool expected_successful_;
1017   string data;
1018 };
1019 
1020 // RedirectTest takes ownership of |http_fetcher|.
RedirectTest(const HttpServer * server,bool expected_successful,const string & url,HttpFetcher * http_fetcher)1021 void RedirectTest(const HttpServer* server,
1022                   bool expected_successful,
1023                   const string& url,
1024                   HttpFetcher* http_fetcher) {
1025   RedirectHttpFetcherTestDelegate delegate(expected_successful);
1026   unique_ptr<HttpFetcher> fetcher(http_fetcher);
1027   fetcher->set_delegate(&delegate);
1028 
1029   MessageLoop::current()->PostTask(
1030       FROM_HERE,
1031       base::Bind(StartTransfer,
1032                  fetcher.get(),
1033                  LocalServerUrlForPath(server->GetPort(), url)));
1034   MessageLoop::current()->Run();
1035   if (expected_successful) {
1036     // verify the data we get back
1037     ASSERT_EQ(static_cast<size_t>(kMediumLength), delegate.data.size());
1038     for (int i = 0; i < kMediumLength; i += 10) {
1039       // Assert so that we don't flood the screen w/ EXPECT errors on failure.
1040       ASSERT_EQ(delegate.data.substr(i, 10), "abcdefghij");
1041     }
1042   }
1043 }
1044 }  // namespace
1045 
TYPED_TEST(HttpFetcherTest,SimpleRedirectTest)1046 TYPED_TEST(HttpFetcherTest, SimpleRedirectTest) {
1047   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
1048     return;
1049 
1050   unique_ptr<HttpServer> server(this->test_.CreateServer());
1051   ASSERT_TRUE(server->started_);
1052 
1053   for (size_t c = 0; c < base::size(kRedirectCodes); ++c) {
1054     const string url = base::StringPrintf(
1055         "/redirect/%d/download/%d", kRedirectCodes[c], kMediumLength);
1056     RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
1057   }
1058 }
1059 
TYPED_TEST(HttpFetcherTest,MaxRedirectTest)1060 TYPED_TEST(HttpFetcherTest, MaxRedirectTest) {
1061   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
1062     return;
1063 
1064   unique_ptr<HttpServer> server(this->test_.CreateServer());
1065   ASSERT_TRUE(server->started_);
1066 
1067   string url;
1068   for (int r = 0; r < kDownloadMaxRedirects; r++) {
1069     url += base::StringPrintf("/redirect/%d",
1070                               kRedirectCodes[r % base::size(kRedirectCodes)]);
1071   }
1072   url += base::StringPrintf("/download/%d", kMediumLength);
1073   RedirectTest(server.get(), true, url, this->test_.NewLargeFetcher());
1074 }
1075 
TYPED_TEST(HttpFetcherTest,BeyondMaxRedirectTest)1076 TYPED_TEST(HttpFetcherTest, BeyondMaxRedirectTest) {
1077   if (this->test_.IsMock() || !this->test_.IsHttpSupported())
1078     return;
1079 
1080   unique_ptr<HttpServer> server(this->test_.CreateServer());
1081   ASSERT_TRUE(server->started_);
1082 
1083   string url;
1084   for (int r = 0; r < kDownloadMaxRedirects + 1; r++) {
1085     url += base::StringPrintf("/redirect/%d",
1086                               kRedirectCodes[r % base::size(kRedirectCodes)]);
1087   }
1088   url += base::StringPrintf("/download/%d", kMediumLength);
1089   RedirectTest(server.get(), false, url, this->test_.NewLargeFetcher());
1090 }
1091 
1092 namespace {
1093 class MultiHttpFetcherTestDelegate : public HttpFetcherDelegate {
1094  public:
MultiHttpFetcherTestDelegate(int expected_response_code)1095   explicit MultiHttpFetcherTestDelegate(int expected_response_code)
1096       : expected_response_code_(expected_response_code) {}
1097 
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)1098   bool ReceivedBytes(HttpFetcher* fetcher,
1099                      const void* bytes,
1100                      size_t length) override {
1101     EXPECT_EQ(fetcher, fetcher_.get());
1102     data.append(reinterpret_cast<const char*>(bytes), length);
1103     return true;
1104   }
1105 
TransferComplete(HttpFetcher * fetcher,bool successful)1106   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1107     EXPECT_EQ(fetcher, fetcher_.get());
1108     EXPECT_EQ(expected_response_code_ != kHttpResponseUndefined, successful);
1109     if (expected_response_code_ != 0)
1110       EXPECT_EQ(expected_response_code_, fetcher->http_response_code());
1111     // Destroy the fetcher (because we're allowed to).
1112     fetcher_.reset(nullptr);
1113     MessageLoop::current()->BreakLoop();
1114   }
1115 
TransferTerminated(HttpFetcher * fetcher)1116   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
1117 
1118   unique_ptr<HttpFetcher> fetcher_;
1119   int expected_response_code_;
1120   string data;
1121 };
1122 
MultiTest(HttpFetcher * fetcher_in,FakeHardware * fake_hardware,const string & url,const vector<pair<off_t,off_t>> & ranges,const string & expected_prefix,size_t expected_size,HttpResponseCode expected_response_code)1123 void MultiTest(HttpFetcher* fetcher_in,
1124                FakeHardware* fake_hardware,
1125                const string& url,
1126                const vector<pair<off_t, off_t>>& ranges,
1127                const string& expected_prefix,
1128                size_t expected_size,
1129                HttpResponseCode expected_response_code) {
1130   MultiHttpFetcherTestDelegate delegate(expected_response_code);
1131   delegate.fetcher_.reset(fetcher_in);
1132 
1133   MultiRangeHttpFetcher* multi_fetcher =
1134       static_cast<MultiRangeHttpFetcher*>(fetcher_in);
1135   ASSERT_TRUE(multi_fetcher);
1136   multi_fetcher->ClearRanges();
1137   for (vector<pair<off_t, off_t>>::const_iterator it = ranges.begin(),
1138                                                   e = ranges.end();
1139        it != e;
1140        ++it) {
1141     string tmp_str = base::StringPrintf("%jd+", it->first);
1142     if (it->second > 0) {
1143       base::StringAppendF(&tmp_str, "%jd", it->second);
1144       multi_fetcher->AddRange(it->first, it->second);
1145     } else {
1146       base::StringAppendF(&tmp_str, "?");
1147       multi_fetcher->AddRange(it->first);
1148     }
1149     LOG(INFO) << "added range: " << tmp_str;
1150   }
1151   fake_hardware->SetIsOfficialBuild(false);
1152   multi_fetcher->set_delegate(&delegate);
1153 
1154   MessageLoop::current()->PostTask(
1155       FROM_HERE, base::Bind(StartTransfer, multi_fetcher, url));
1156   MessageLoop::current()->Run();
1157 
1158   EXPECT_EQ(expected_size, delegate.data.size());
1159   EXPECT_EQ(expected_prefix,
1160             string(delegate.data.data(), expected_prefix.size()));
1161 }
1162 }  // namespace
1163 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherSimpleTest)1164 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherSimpleTest) {
1165   if (!this->test_.IsMulti())
1166     return;
1167 
1168   unique_ptr<HttpServer> server(this->test_.CreateServer());
1169   ASSERT_TRUE(server->started_);
1170 
1171   vector<pair<off_t, off_t>> ranges;
1172   ranges.push_back(make_pair(0, 25));
1173   ranges.push_back(make_pair(99, 17));
1174   MultiTest(this->test_.NewLargeFetcher(),
1175             this->test_.fake_hardware(),
1176             this->test_.BigUrl(server->GetPort()),
1177             ranges,
1178             "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1179             25 + 17,
1180             this->test_.IsFileFetcher() ? kHttpResponseOk
1181                                         : kHttpResponsePartialContent);
1182 }
1183 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherUnspecifiedEndTest)1184 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherUnspecifiedEndTest) {
1185   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1186     return;
1187 
1188   unique_ptr<HttpServer> server(this->test_.CreateServer());
1189   ASSERT_TRUE(server->started_);
1190 
1191   vector<pair<off_t, off_t>> ranges;
1192   ranges.push_back(make_pair(0, 25));
1193   ranges.push_back(make_pair(99, 0));
1194   MultiTest(this->test_.NewLargeFetcher(),
1195             this->test_.fake_hardware(),
1196             this->test_.BigUrl(server->GetPort()),
1197             ranges,
1198             "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1199             kBigLength - (99 - 25),
1200             kHttpResponsePartialContent);
1201 }
1202 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherLengthLimitTest)1203 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherLengthLimitTest) {
1204   if (!this->test_.IsMulti())
1205     return;
1206 
1207   unique_ptr<HttpServer> server(this->test_.CreateServer());
1208   ASSERT_TRUE(server->started_);
1209 
1210   vector<pair<off_t, off_t>> ranges;
1211   ranges.push_back(make_pair(0, 24));
1212   MultiTest(this->test_.NewLargeFetcher(),
1213             this->test_.fake_hardware(),
1214             this->test_.BigUrl(server->GetPort()),
1215             ranges,
1216             "abcdefghijabcdefghijabcd",
1217             24,
1218             this->test_.IsFileFetcher() ? kHttpResponseOk
1219                                         : kHttpResponsePartialContent);
1220 }
1221 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherMultiEndTest)1222 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherMultiEndTest) {
1223   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1224     return;
1225 
1226   unique_ptr<HttpServer> server(this->test_.CreateServer());
1227   ASSERT_TRUE(server->started_);
1228 
1229   vector<pair<off_t, off_t>> ranges;
1230   ranges.push_back(make_pair(kBigLength - 2, 0));
1231   ranges.push_back(make_pair(kBigLength - 3, 0));
1232   MultiTest(this->test_.NewLargeFetcher(),
1233             this->test_.fake_hardware(),
1234             this->test_.BigUrl(server->GetPort()),
1235             ranges,
1236             "ijhij",
1237             5,
1238             kHttpResponsePartialContent);
1239 }
1240 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherInsufficientTest)1241 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherInsufficientTest) {
1242   if (!this->test_.IsMulti())
1243     return;
1244 
1245   unique_ptr<HttpServer> server(this->test_.CreateServer());
1246   ASSERT_TRUE(server->started_);
1247 
1248   vector<pair<off_t, off_t>> ranges;
1249   ranges.push_back(make_pair(kBigLength - 2, 4));
1250   for (int i = 0; i < 2; ++i) {
1251     LOG(INFO) << "i = " << i;
1252     MultiTest(this->test_.NewLargeFetcher(),
1253               this->test_.fake_hardware(),
1254               this->test_.BigUrl(server->GetPort()),
1255               ranges,
1256               "ij",
1257               2,
1258               kHttpResponseUndefined);
1259     ranges.push_back(make_pair(0, 5));
1260   }
1261 }
1262 
1263 // Issue #18143: when a fetch of a secondary chunk out of a chain, then it
1264 // should retry with other proxies listed before giving up.
1265 //
1266 // (1) successful recovery: The offset fetch will fail twice but succeed with
1267 // the third proxy.
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherErrorIfOffsetRecoverableTest)1268 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetRecoverableTest) {
1269   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1270     return;
1271 
1272   unique_ptr<HttpServer> server(this->test_.CreateServer());
1273   ASSERT_TRUE(server->started_);
1274 
1275   vector<pair<off_t, off_t>> ranges;
1276   ranges.push_back(make_pair(0, 25));
1277   ranges.push_back(make_pair(99, 0));
1278   MultiTest(this->test_.NewLargeFetcher(3),
1279             this->test_.fake_hardware(),
1280             LocalServerUrlForPath(
1281                 server->GetPort(),
1282                 base::StringPrintf("/error-if-offset/%d/2", kBigLength)),
1283             ranges,
1284             "abcdefghijabcdefghijabcdejabcdefghijabcdef",
1285             kBigLength - (99 - 25),
1286             kHttpResponsePartialContent);
1287 }
1288 
1289 // (2) unsuccessful recovery: The offset fetch will fail repeatedly.  The
1290 // fetcher will signal a (failed) completed transfer to the delegate.
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherErrorIfOffsetUnrecoverableTest)1291 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherErrorIfOffsetUnrecoverableTest) {
1292   if (!this->test_.IsMulti() || this->test_.IsFileFetcher())
1293     return;
1294 
1295   unique_ptr<HttpServer> server(this->test_.CreateServer());
1296   ASSERT_TRUE(server->started_);
1297 
1298   vector<pair<off_t, off_t>> ranges;
1299   ranges.push_back(make_pair(0, 25));
1300   ranges.push_back(make_pair(99, 0));
1301   MultiTest(this->test_.NewLargeFetcher(2),
1302             this->test_.fake_hardware(),
1303             LocalServerUrlForPath(
1304                 server->GetPort(),
1305                 base::StringPrintf("/error-if-offset/%d/3", kBigLength)),
1306             ranges,
1307             "abcdefghijabcdefghijabcde",  // only received the first chunk
1308             25,
1309             kHttpResponseUndefined);
1310 }
1311 
1312 namespace {
1313 // This HttpFetcherDelegate calls TerminateTransfer at a configurable point.
1314 class MultiHttpFetcherTerminateTestDelegate : public HttpFetcherDelegate {
1315  public:
MultiHttpFetcherTerminateTestDelegate(size_t terminate_trigger_bytes)1316   explicit MultiHttpFetcherTerminateTestDelegate(size_t terminate_trigger_bytes)
1317       : terminate_trigger_bytes_(terminate_trigger_bytes) {}
1318 
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)1319   bool ReceivedBytes(HttpFetcher* fetcher,
1320                      const void* bytes,
1321                      size_t length) override {
1322     LOG(INFO) << "ReceivedBytes, " << length << " bytes.";
1323     EXPECT_EQ(fetcher, fetcher_.get());
1324     bool should_terminate = false;
1325     if (bytes_downloaded_ < terminate_trigger_bytes_ &&
1326         bytes_downloaded_ + length >= terminate_trigger_bytes_) {
1327       MessageLoop::current()->PostTask(
1328           FROM_HERE,
1329           base::Bind(&HttpFetcher::TerminateTransfer,
1330                      base::Unretained(fetcher_.get())));
1331       should_terminate = true;
1332     }
1333     bytes_downloaded_ += length;
1334     return !should_terminate;
1335   }
1336 
TransferComplete(HttpFetcher * fetcher,bool successful)1337   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1338     ADD_FAILURE() << "TransferComplete called but expected a failure";
1339     // Destroy the fetcher (because we're allowed to).
1340     fetcher_.reset(nullptr);
1341     MessageLoop::current()->BreakLoop();
1342   }
1343 
TransferTerminated(HttpFetcher * fetcher)1344   void TransferTerminated(HttpFetcher* fetcher) override {
1345     // Destroy the fetcher (because we're allowed to).
1346     fetcher_.reset(nullptr);
1347     MessageLoop::current()->BreakLoop();
1348   }
1349 
1350   unique_ptr<HttpFetcher> fetcher_;
1351   size_t bytes_downloaded_{0};
1352   size_t terminate_trigger_bytes_;
1353 };
1354 }  // namespace
1355 
TYPED_TEST(HttpFetcherTest,MultiHttpFetcherTerminateBetweenRangesTest)1356 TYPED_TEST(HttpFetcherTest, MultiHttpFetcherTerminateBetweenRangesTest) {
1357   if (!this->test_.IsMulti())
1358     return;
1359   const size_t kRangeTrigger = 1000;
1360   MultiHttpFetcherTerminateTestDelegate delegate(kRangeTrigger);
1361 
1362   unique_ptr<HttpServer> server(this->test_.CreateServer());
1363   ASSERT_TRUE(server->started_);
1364 
1365   MultiRangeHttpFetcher* multi_fetcher =
1366       static_cast<MultiRangeHttpFetcher*>(this->test_.NewLargeFetcher());
1367   ASSERT_TRUE(multi_fetcher);
1368   // Transfer ownership of the fetcher to the delegate.
1369   delegate.fetcher_.reset(multi_fetcher);
1370   multi_fetcher->set_delegate(&delegate);
1371 
1372   multi_fetcher->ClearRanges();
1373   multi_fetcher->AddRange(45, kRangeTrigger);
1374   multi_fetcher->AddRange(2000, 100);
1375 
1376   this->test_.fake_hardware()->SetIsOfficialBuild(false);
1377 
1378   StartTransfer(multi_fetcher, this->test_.BigUrl(server->GetPort()));
1379   MessageLoop::current()->Run();
1380 
1381   // Check that the delegate made it to the trigger point.
1382   EXPECT_EQ(kRangeTrigger, delegate.bytes_downloaded_);
1383 }
1384 
1385 namespace {
1386 class BlockedTransferTestDelegate : public HttpFetcherDelegate {
1387  public:
ReceivedBytes(HttpFetcher * fetcher,const void * bytes,size_t length)1388   bool ReceivedBytes(HttpFetcher* fetcher,
1389                      const void* bytes,
1390                      size_t length) override {
1391     ADD_FAILURE();
1392     return true;
1393   }
TransferComplete(HttpFetcher * fetcher,bool successful)1394   void TransferComplete(HttpFetcher* fetcher, bool successful) override {
1395     EXPECT_FALSE(successful);
1396     MessageLoop::current()->BreakLoop();
1397   }
TransferTerminated(HttpFetcher * fetcher)1398   void TransferTerminated(HttpFetcher* fetcher) override { ADD_FAILURE(); }
1399 };
1400 
BlockedTransferTestHelper(AnyHttpFetcherTest * fetcher_test,bool is_official_build)1401 void BlockedTransferTestHelper(AnyHttpFetcherTest* fetcher_test,
1402                                bool is_official_build) {
1403   if (fetcher_test->IsMock() || fetcher_test->IsMulti())
1404     return;
1405 
1406   unique_ptr<HttpServer> server(fetcher_test->CreateServer());
1407   ASSERT_TRUE(server->started_);
1408 
1409   BlockedTransferTestDelegate delegate;
1410   unique_ptr<HttpFetcher> fetcher(fetcher_test->NewLargeFetcher());
1411   LOG(INFO) << "is_official_build: " << is_official_build;
1412   // NewLargeFetcher creates the HttpFetcher* with a FakeSystemState.
1413   fetcher_test->fake_hardware()->SetIsOfficialBuild(is_official_build);
1414   fetcher->set_delegate(&delegate);
1415 
1416   MessageLoop::current()->PostTask(
1417       FROM_HERE,
1418       base::Bind(
1419           StartTransfer,
1420           fetcher.get(),
1421           LocalServerUrlForPath(server->GetPort(),
1422                                 fetcher_test->SmallUrl(server->GetPort()))));
1423   MessageLoop::current()->Run();
1424 }
1425 }  // namespace
1426 
TYPED_TEST(HttpFetcherTest,BlockedTransferTest)1427 TYPED_TEST(HttpFetcherTest, BlockedTransferTest) {
1428   BlockedTransferTestHelper(&this->test_, false);
1429 }
1430 
TYPED_TEST(HttpFetcherTest,BlockedTransferOfficialBuildTest)1431 TYPED_TEST(HttpFetcherTest, BlockedTransferOfficialBuildTest) {
1432   BlockedTransferTestHelper(&this->test_, true);
1433 }
1434 
1435 }  // namespace chromeos_update_engine
1436