1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.util; 17 18 import com.android.tradefed.log.ITestLogger; 19 import com.android.tradefed.log.LogUtil.CLog; 20 import com.android.tradefed.result.FileInputStreamSource; 21 import com.android.tradefed.result.InputStreamSource; 22 import com.android.tradefed.result.LogDataType; 23 24 import org.apache.commons.compress.archivers.ArchiveException; 25 import org.apache.commons.compress.archivers.ArchiveStreamFactory; 26 import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 27 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 28 import org.apache.commons.compress.utils.IOUtils; 29 30 import java.io.File; 31 import java.io.FileInputStream; 32 import java.io.FileNotFoundException; 33 import java.io.FileOutputStream; 34 import java.io.IOException; 35 import java.io.InputStream; 36 import java.io.OutputStream; 37 import java.util.Arrays; 38 import java.util.LinkedList; 39 import java.util.List; 40 import java.util.zip.GZIPInputStream; 41 import java.util.zip.GZIPOutputStream; 42 43 /** 44 * Utility to manipulate a tar file. It wraps the commons-compress in order to provide tar support. 45 */ 46 public class TarUtil { 47 48 private static final byte[] GZIP_SIGNATURE = {0x1f, (byte) 0x8b}; 49 50 /** 51 * Determine whether a file is a gzip. 52 * 53 * @param file the file to check. 54 * @return whether the file is a gzip. 55 * @throws IOException if the file could not be read. 56 */ isGzip(File file)57 public static boolean isGzip(File file) throws IOException { 58 byte[] signature = new byte[GZIP_SIGNATURE.length]; 59 try (InputStream stream = new FileInputStream(file)) { 60 if (stream.read(signature) != signature.length) { 61 return false; 62 } 63 } 64 return Arrays.equals(GZIP_SIGNATURE, signature); 65 } 66 67 /** 68 * Untar a tar file into a directory. 69 * tar.gz file need to up {@link #unGzip(File, File)} first. 70 * 71 * @param inputFile The tar file to extract 72 * @param outputDir the directory where to put the extracted files. 73 * @return The list of {@link File} untarred. 74 * @throws FileNotFoundException 75 * @throws IOException 76 */ unTar(final File inputFile, final File outputDir)77 public static List<File> unTar(final File inputFile, final File outputDir) 78 throws FileNotFoundException, IOException { 79 CLog.i(String.format("Untaring %s to dir %s.", inputFile.getAbsolutePath(), 80 outputDir.getAbsolutePath())); 81 final List<File> untaredFiles = new LinkedList<File>(); 82 final InputStream is = new FileInputStream(inputFile); 83 TarArchiveInputStream debInputStream = null; 84 try { 85 debInputStream = (TarArchiveInputStream) 86 new ArchiveStreamFactory().createArchiveInputStream("tar", is); 87 TarArchiveEntry entry = null; 88 while ((entry = (TarArchiveEntry)debInputStream.getNextEntry()) != null) { 89 final File outputFile = new File(outputDir, entry.getName()); 90 if (entry.isDirectory()) { 91 CLog.i(String.format("Attempting to write output directory %s.", 92 outputFile.getAbsolutePath())); 93 if (!outputFile.exists()) { 94 CLog.i(String.format("Attempting to create output directory %s.", 95 outputFile.getAbsolutePath())); 96 if (!outputFile.mkdirs()) { 97 throw new IllegalStateException( 98 String.format("Couldn't create directory %s.", 99 outputFile.getAbsolutePath())); 100 } 101 } 102 } else { 103 CLog.i(String.format("Creating output file %s.", outputFile.getAbsolutePath())); 104 final File parent = outputFile.getParentFile(); 105 if (parent != null && !parent.exists()) { 106 if (!parent.mkdirs()) { 107 throw new IOException( 108 String.format( 109 "Couldn't create directory %s.", 110 parent.getAbsolutePath())); 111 } 112 } 113 final OutputStream outputFileStream = new FileOutputStream(outputFile); 114 IOUtils.copy(debInputStream, outputFileStream); 115 StreamUtil.close(outputFileStream); 116 } 117 untaredFiles.add(outputFile); 118 } 119 } catch (ArchiveException ae) { 120 // We rethrow the ArchiveException through a more generic one. 121 throw new IOException(ae); 122 } finally { 123 StreamUtil.close(debInputStream); 124 StreamUtil.close(is); 125 } 126 return untaredFiles; 127 } 128 129 /** 130 * UnGZip a file: a tar.gz file will become a tar file. 131 * 132 * @param inputFile The {@link File} to ungzip 133 * @param outputDir The directory where to put the ungzipped file. 134 * @return a {@link File} pointing to the ungzipped file. 135 * @throws FileNotFoundException 136 * @throws IOException 137 */ unGzip(final File inputFile, final File outputDir)138 public static File unGzip(final File inputFile, final File outputDir) 139 throws FileNotFoundException, IOException { 140 CLog.i(String.format("Ungzipping %s to dir %s.", inputFile.getAbsolutePath(), 141 outputDir.getAbsolutePath())); 142 // rename '-3' to remove the '.gz' extension. 143 final File outputFile = new File(outputDir, inputFile.getName().substring(0, 144 inputFile.getName().length() - 3)); 145 GZIPInputStream in = null; 146 FileOutputStream out = null; 147 try { 148 in = new GZIPInputStream(new FileInputStream(inputFile)); 149 out = new FileOutputStream(outputFile); 150 IOUtils.copy(in, out); 151 } finally { 152 StreamUtil.close(in); 153 StreamUtil.close(out); 154 } 155 return outputFile; 156 } 157 158 /** 159 * Utility function to gzip (.gz) a file. the .gz extension will be added to base file name. 160 * 161 * @param inputFile the {@link File} to be gzipped. 162 * @return the gzipped file. 163 * @throws IOException 164 */ gzip(final File inputFile)165 public static File gzip(final File inputFile) throws IOException { 166 File outputFile = FileUtil.createTempFile(inputFile.getName(), ".gz"); 167 GZIPOutputStream out = null; 168 FileInputStream in = null; 169 try { 170 out = new GZIPOutputStream(new FileOutputStream(outputFile)); 171 in = new FileInputStream(inputFile); 172 IOUtils.copy(in, out); 173 } catch (IOException e) { 174 // delete the tmp file if we failed to gzip. 175 FileUtil.deleteFile(outputFile); 176 throw e; 177 } finally { 178 StreamUtil.close(in); 179 StreamUtil.close(out); 180 } 181 return outputFile; 182 } 183 184 /** 185 * Untar and ungzip a tar.gz file to a temp directory. 186 * 187 * @param targzFile the tar.gz file to extract. 188 * @param nameHint the prefix for the temp directory. 189 * @return the temp directory. 190 * @throws FileNotFoundException 191 * @throws IOException 192 */ extractTarGzipToTemp(File targzFile, String nameHint)193 public static File extractTarGzipToTemp(File targzFile, String nameHint) 194 throws FileNotFoundException, IOException { 195 File unGzipDir = null; 196 File unTarDir = null; 197 try { 198 unGzipDir = FileUtil.createTempDir("extractTarGzip"); 199 File tarFile = TarUtil.unGzip(targzFile, unGzipDir); 200 unTarDir = FileUtil.createTempDir(nameHint); 201 TarUtil.unTar(tarFile, unTarDir); 202 return unTarDir; 203 } catch (IOException e) { 204 FileUtil.recursiveDelete(unTarDir); 205 throw e; 206 } finally { 207 FileUtil.recursiveDelete(unGzipDir); 208 } 209 } 210 211 /** 212 * Helper to extract and log to the reporters a tar gz file and its content 213 * 214 * @param listener the {@link ITestLogger} where to log the files. 215 * @param targzFile the tar.gz {@link File} that needs its content log. 216 * @param baseName the base name under which the files will be found. 217 */ extractAndLog(ITestLogger listener, File targzFile, String baseName)218 public static void extractAndLog(ITestLogger listener, File targzFile, String baseName) 219 throws FileNotFoundException, IOException { 220 // First upload the tar.gz file 221 InputStreamSource inputStream = null; 222 try { 223 inputStream = new FileInputStreamSource(targzFile); 224 listener.testLog(baseName, LogDataType.TAR_GZ, inputStream); 225 } finally { 226 StreamUtil.cancel(inputStream); 227 } 228 229 // extract and upload internal files. 230 File dir = FileUtil.createTempDir("tmp_tar_dir"); 231 File ungzipLog = null; 232 try { 233 ungzipLog = TarUtil.unGzip(targzFile, dir); 234 List<File> logs = TarUtil.unTar(ungzipLog, dir); 235 for (File f : logs) { 236 InputStreamSource s = null; 237 try { 238 s = new FileInputStreamSource(f); 239 listener.testLog(String.format("%s_%s", baseName, f.getName()), 240 LogDataType.TEXT, s); 241 } finally { 242 StreamUtil.cancel(s); 243 } 244 } 245 } finally { 246 FileUtil.deleteFile(ungzipLog); 247 FileUtil.recursiveDelete(dir); 248 } 249 } 250 } 251