1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.util;
18 
19 import com.android.tradefed.build.BuildRetrievalError;
20 
21 import com.google.api.client.googleapis.batch.BatchCallback;
22 import com.google.api.client.googleapis.batch.BatchRequest;
23 import com.google.api.client.http.HttpHeaders;
24 import com.google.api.client.http.InputStreamContent;
25 import com.google.api.services.storage.Storage;
26 import com.google.api.services.storage.Storage.Objects.List;
27 import com.google.api.services.storage.model.Objects;
28 import com.google.api.services.storage.model.StorageObject;
29 
30 import org.junit.After;
31 import org.junit.Assert;
32 import org.junit.Before;
33 import org.junit.Test;
34 import org.junit.runner.RunWith;
35 import org.junit.runners.JUnit4;
36 
37 import java.io.ByteArrayInputStream;
38 import java.io.File;
39 import java.io.IOException;
40 import java.io.InputStream;
41 import java.nio.file.Paths;
42 import java.util.Collections;
43 
44 /** {@link GCSFileDownloader} functional test. */
45 @RunWith(JUnit4.class)
46 public class GCSFileDownloaderFuncTest {
47 
48     private static final String BUCKET_NAME = "tradefed_function_test";
49     private static final String FILE_NAME1 = "a_host_config.xml";
50     private static final String FILE_NAME2 = "file2.txt";
51     private static final String FILE_NAME3 = "file3.txt";
52     private static final String FILE_NAME4 = "file4.txt";
53     private static final String FOLDER_NAME1 = "folder1";
54     private static final String FOLDER_NAME2 = "folder2";
55     private static final String FILE_CONTENT = "Hello World!";
56 
57     private GCSFileDownloader mDownloader;
58     private String mRemoteRoot;
59     private File mLocalRoot;
60     private Storage mStorage;
61 
createFile( Storage storage, String content, String bucketName, String... pathSegs)62     private static void createFile(
63             Storage storage, String content, String bucketName, String... pathSegs)
64             throws IOException {
65         String path = String.join("/", pathSegs);
66         StorageObject object = new StorageObject();
67         object.setName(path);
68         storage.objects()
69                 .insert(
70                         bucketName,
71                         object,
72                         new InputStreamContent(null, new ByteArrayInputStream(content.getBytes())))
73                 .execute();
74     }
75 
76     @Before
setUp()77     public void setUp() throws IOException {
78         File tempFile =
79                 FileUtil.createTempFile(GCSFileDownloaderFuncTest.class.getSimpleName(), "");
80         mRemoteRoot = tempFile.getName();
81         FileUtil.deleteFile(tempFile);
82         mDownloader =
83                 new GCSFileDownloader() {
84 
85                     @Override
86                     File createTempFile(String remoteFilePath, File rootDir)
87                             throws BuildRetrievalError {
88                         try {
89                             File tmpFile =
90                                     FileUtil.createTempFileForRemote(remoteFilePath, mLocalRoot);
91                             tmpFile.delete();
92                             return tmpFile;
93                         } catch (IOException e) {
94                             throw new BuildRetrievalError(e.getMessage(), e);
95                         }
96                     }
97                 };
98         mStorage =
99                 mDownloader.getStorage(
100                         Collections.singleton(
101                                 "https://www.googleapis.com/auth/devstorage.read_write"));
102         createFile(mStorage, FILE_CONTENT, BUCKET_NAME, mRemoteRoot, FILE_NAME1);
103         createFile(mStorage, FILE_NAME2, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME2);
104         createFile(mStorage, FILE_NAME3, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3);
105         // Create a special case condition where folder name is also a file name.
106         createFile(mStorage, FILE_NAME3, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME2);
107         createFile(
108                 mStorage,
109                 FILE_NAME4,
110                 BUCKET_NAME,
111                 mRemoteRoot,
112                 FOLDER_NAME1,
113                 FOLDER_NAME2,
114                 FILE_NAME4);
115         mLocalRoot = FileUtil.createTempDir(GCSFileDownloaderFuncTest.class.getSimpleName());
116     }
117 
118     @After
tearDown()119     public void tearDown() throws IOException {
120         FileUtil.recursiveDelete(mLocalRoot);
121         String pageToken = null;
122         BatchRequest batchRequest = mStorage.batch();
123 
124         while (true) {
125             List listOperation = mStorage.objects().list(BUCKET_NAME).setPrefix(mRemoteRoot);
126             if (pageToken == null) {
127                 listOperation.setPageToken(pageToken);
128             }
129             Objects objects = listOperation.execute();
130             for (StorageObject object : objects.getItems()) {
131                 batchRequest.queue(
132                         mStorage.objects().delete(BUCKET_NAME, object.getName()).buildHttpRequest(),
133                         Void.class,
134                         IOException.class,
135                         new BatchCallback<Void, IOException>() {
136                             @Override
137                             public void onSuccess(Void arg0, HttpHeaders arg1) throws IOException {}
138 
139                             @Override
140                             public void onFailure(IOException e, HttpHeaders arg1)
141                                     throws IOException {
142                                 throw e;
143                             }
144                         });
145             }
146             pageToken = objects.getNextPageToken();
147             if (pageToken == null) {
148                 batchRequest.execute();
149                 return;
150             }
151         }
152     }
153 
154     @Test
testDownloadFile_streamOutput()155     public void testDownloadFile_streamOutput() throws Exception {
156         InputStream inputStream =
157                 mDownloader.downloadFile(BUCKET_NAME, mRemoteRoot + "/" + FILE_NAME1);
158         String content = StreamUtil.getStringFromStream(inputStream);
159         Assert.assertEquals(FILE_CONTENT, content);
160         inputStream.reset();
161     }
162 
163     @Test
testDownloadFile_streamOutput_notExist()164     public void testDownloadFile_streamOutput_notExist() throws Exception {
165         try {
166             mDownloader.downloadFile(BUCKET_NAME, mRemoteRoot + "/" + "non_exist_file");
167             Assert.fail("Should throw IOException.");
168         } catch (IOException e) {
169             // Expect IOException
170         }
171     }
172 
173     @Test
testGetRemoteFileMetaData()174     public void testGetRemoteFileMetaData() throws Exception {
175         String filename = mRemoteRoot + "/" + FILE_NAME1;
176         StorageObject object = mDownloader.getRemoteFileMetaData(BUCKET_NAME, filename);
177         Assert.assertEquals(filename, object.getName());
178     }
179 
180     @Test
testGetRemoteFileMetaData_notExist()181     public void testGetRemoteFileMetaData_notExist() throws Exception {
182         String filename = mRemoteRoot + "/" + "not_exist";
183         StorageObject object = mDownloader.getRemoteFileMetaData(BUCKET_NAME, filename);
184         Assert.assertNull(object);
185     }
186 
187     @Test
testIsRemoteFolder()188     public void testIsRemoteFolder() throws Exception {
189         Assert.assertFalse(
190                 mDownloader.isRemoteFolder(
191                         BUCKET_NAME, Paths.get(mRemoteRoot, FILE_NAME1).toString()));
192         Assert.assertTrue(
193                 mDownloader.isRemoteFolder(
194                         BUCKET_NAME, Paths.get(mRemoteRoot, FOLDER_NAME1).toString()));
195     }
196 
197     @Test
testDownloadFile()198     public void testDownloadFile() throws Exception {
199         File localFile =
200                 mDownloader.downloadFile(
201                         String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1));
202         String content = FileUtil.readStringFromFile(localFile);
203         Assert.assertEquals(FILE_CONTENT, content);
204     }
205 
206     @Test
testDownloadFile_nonExist()207     public void testDownloadFile_nonExist() throws Exception {
208         try {
209             mDownloader.downloadFile(
210                     String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, "non_exist_file"));
211             Assert.fail("Should throw BuildRetrievalError.");
212         } catch (BuildRetrievalError e) {
213             // Expect BuildRetrievalError
214         }
215     }
216 
217     @Test
testDownloadFile_folder()218     public void testDownloadFile_folder() throws Exception {
219         File localFile =
220                 mDownloader.downloadFile(
221                         String.format("gs://%s/%s/%s/", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1));
222         checkDownloadedFolder(localFile);
223     }
224 
225     @Test
testDownloadFile_folderNotsanitize()226     public void testDownloadFile_folderNotsanitize() throws Exception {
227         File localFile =
228                 mDownloader.downloadFile(
229                         String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1));
230         checkDownloadedFolder(localFile);
231     }
232 
checkDownloadedFolder(File localFile)233     private void checkDownloadedFolder(File localFile) throws Exception {
234         Assert.assertTrue(localFile.isDirectory());
235         Assert.assertEquals(4, localFile.list().length);
236         for (String filename : localFile.list()) {
237             if (filename.equals(FILE_NAME2)) {
238                 Assert.assertEquals(
239                         FILE_NAME2,
240                         FileUtil.readStringFromFile(
241                                 new File(localFile.getAbsolutePath(), filename)));
242             } else if (filename.equals(FILE_NAME3)) {
243                 Assert.assertEquals(
244                         FILE_NAME3,
245                         FileUtil.readStringFromFile(
246                                 new File(localFile.getAbsolutePath(), filename)));
247             } else if (filename.equals(FOLDER_NAME2 + "_folder")) {
248                 File subFolder = new File(localFile.getAbsolutePath(), filename);
249                 Assert.assertTrue(subFolder.isDirectory());
250                 Assert.assertEquals(1, subFolder.list().length);
251                 Assert.assertEquals(
252                         FILE_NAME4,
253                         FileUtil.readStringFromFile(
254                                 new File(subFolder.getAbsolutePath(), subFolder.list()[0])));
255             } else if (filename.equals(FOLDER_NAME2)) {
256                 File fileWithFolderName = new File(localFile.getAbsolutePath(), filename);
257                 Assert.assertTrue(fileWithFolderName.isFile());
258             } else {
259                 Assert.assertTrue(String.format("Unknonwn file %s", filename), false);
260             }
261         }
262     }
263 
264     @Test
testDownloadFile_folder_nonExist()265     public void testDownloadFile_folder_nonExist() throws Exception {
266         try {
267             mDownloader.downloadFile(
268                     String.format("gs://%s/%s/%s/", BUCKET_NAME, "mRemoteRoot", "nonExistFolder"));
269             Assert.fail("Should throw BuildRetrievalError.");
270         } catch (BuildRetrievalError e) {
271             // Expect BuildRetrievalError
272         }
273     }
274 
275     @Test
testCheckFreshness()276     public void testCheckFreshness() throws Exception {
277         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
278         File localFile = mDownloader.downloadFile(remotePath);
279         Assert.assertTrue(mDownloader.isFresh(localFile, remotePath));
280     }
281 
282     @Test
testCheckFreshness_notExist()283     public void testCheckFreshness_notExist() throws Exception {
284         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
285         Assert.assertFalse(mDownloader.isFresh(new File("/not/exist"), remotePath));
286     }
287 
288     @Test
testCheckFreshness_folderNotExist()289     public void testCheckFreshness_folderNotExist() throws Exception {
290         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
291         Assert.assertFalse(mDownloader.isFresh(new File("/not/exist"), remotePath));
292     }
293 
294     @Test
testCheckFreshness_remoteNotExist()295     public void testCheckFreshness_remoteNotExist() throws Exception {
296         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
297         String remoteNotExistPath = String.format("gs://%s/%s/no_exist", BUCKET_NAME, mRemoteRoot);
298         File localFile = mDownloader.downloadFile(remotePath);
299         Assert.assertFalse(mDownloader.isFresh(localFile, remoteNotExistPath));
300     }
301 
302     @Test
testCheckFreshness_remoteFolderNotExist()303     public void testCheckFreshness_remoteFolderNotExist() throws Exception {
304         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
305         String remoteNotExistPath = String.format("gs://%s/%s/no_exist/", BUCKET_NAME, mRemoteRoot);
306         File localFolder = mDownloader.downloadFile(remotePath);
307         Assert.assertFalse(mDownloader.isFresh(localFolder, remoteNotExistPath));
308     }
309 
310     @Test
testCheckFreshness_notFresh()311     public void testCheckFreshness_notFresh() throws Exception {
312         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
313         File localFile = mDownloader.downloadFile(remotePath);
314         // Change the remote file.
315         createFile(mStorage, "New content.", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
316         Assert.assertFalse(mDownloader.isFresh(localFile, remotePath));
317     }
318 
319     @Test
testCheckFreshness_folder()320     public void testCheckFreshness_folder() throws Exception {
321         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
322         File localFolder = mDownloader.downloadFile(remotePath);
323         Assert.assertTrue(mDownloader.isFresh(localFolder, remotePath));
324     }
325 
326     @Test
testCheckFreshness_folder_addFile()327     public void testCheckFreshness_folder_addFile() throws Exception {
328         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
329         File localFolder = mDownloader.downloadFile(remotePath);
330         createFile(
331                 mStorage,
332                 "A new file",
333                 BUCKET_NAME,
334                 mRemoteRoot,
335                 FOLDER_NAME1,
336                 FOLDER_NAME2,
337                 "new_file.txt");
338         Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath));
339     }
340 
341     @Test
testCheckFreshness_folder_removeFile()342     public void testCheckFreshness_folder_removeFile() throws Exception {
343         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
344         File localFolder = mDownloader.downloadFile(remotePath);
345         mStorage.objects()
346                 .delete(BUCKET_NAME, Paths.get(mRemoteRoot, FOLDER_NAME1, FILE_NAME3).toString())
347                 .execute();
348         Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath));
349     }
350 
351     @Test
testCheckFreshness_folder_changeFile()352     public void testCheckFreshness_folder_changeFile() throws Exception {
353         String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
354         File localFolder = mDownloader.downloadFile(remotePath);
355         createFile(mStorage, "New content", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3);
356         Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath));
357     }
358 }
359