1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.hadoop.fs;
20
21 import java.io;
22 import java.util.Enumeration;
23 import java.util.zip.ZipEntry;
24 import java.util.zip.ZipFile;
25 import java.net.Socket;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.util.StringUtils;
29
30 /**
31 * A collection of file-processing util methods
32 */
33 public class FileUtil {
34
35 /**
36 * Delete a directory and all its contents. If
37 * we return false, the directory may be partially-deleted.
38 */
39 public static boolean fullyDelete(File dir) throws IOException {
40 File contents[] = dir.listFiles();
41 if (contents != null) {
42 for (int i = 0; i < contents.length; i++) {
43 if (contents[i].isFile()) {
44 if (!contents[i].delete()) {
45 return false;
46 }
47 } else {
48 //try deleting the directory
49 // this might be a symlink
50 boolean b = false;
51 b = contents[i].delete();
52 if (b){
53 //this was indeed a symlink or an empty directory
54 continue;
55 }
56 // if not an empty directory or symlink let
57 // fullydelete handle it.
58 if (!fullyDelete(contents[i])) {
59 return false;
60 }
61 }
62 }
63 }
64 return dir.delete();
65 }
66
67 /**
68 * Recursively delete a directory.
69 *
70 * @param fs {@link FileSystem} on which the path is present
71 * @param dir directory to recursively delete
72 * @throws IOException
73 */
74 public static void fullyDelete(FileSystem fs, Path dir)
75 throws IOException {
76 Path[] paths = fs.listPaths(dir);
77 if (paths != null) {
78 for (Path p : paths) {
79 if (fs.isFile(p)) {
80 fs.delete(p);
81 } else {
82 fullyDelete(fs, p);
83 }
84 }
85 }
86 fs.delete(dir);
87 }
88
89 //
90 // If the destination is a subdirectory of the source, then
91 // generate exception
92 //
93 private static void checkDependencies(FileSystem srcFS,
94 Path src,
95 FileSystem dstFS,
96 Path dst)
97 throws IOException {
98 if (srcFS == dstFS) {
99 String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
100 String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
101 if (dstq.startsWith(srcq)) {
102 if (srcq.length() == dstq.length()) {
103 throw new IOException("Cannot copy " + src + " to itself.");
104 } else {
105 throw new IOException("Cannot copy " + src + " to its subdirectory " +
106 dst);
107 }
108 }
109 }
110 }
111
112 /** Copy files between FileSystems. */
113 public static boolean copy(FileSystem srcFS, Path src,
114 FileSystem dstFS, Path dst,
115 boolean deleteSource,
116 Configuration conf) throws IOException {
117 dst = checkDest(src.getName(), dstFS, dst);
118
119 if (srcFS.isDirectory(src)) {
120 checkDependencies(srcFS, src, dstFS, dst);
121 if (!dstFS.mkdirs(dst)) {
122 return false;
123 }
124 Path contents[] = srcFS.listPaths(src);
125 for (int i = 0; i < contents.length; i++) {
126 copy(srcFS, contents[i], dstFS, new Path(dst, contents[i].getName()),
127 deleteSource, conf);
128 }
129 } else if (srcFS.isFile(src)) {
130 InputStream in = srcFS.open(src);
131 try {
132 OutputStream out = dstFS.create(dst);
133 copyContent(in, out, conf);
134 } finally {
135 in.close();
136 }
137 } else {
138 throw new IOException(src.toString() + ": No such file or directory");
139 }
140 if (deleteSource) {
141 return srcFS.delete(src);
142 } else {
143 return true;
144 }
145
146 }
147
148 /** Copy all files in a directory to one output file (merge). */
149 public static boolean copyMerge(FileSystem srcFS, Path srcDir,
150 FileSystem dstFS, Path dstFile,
151 boolean deleteSource,
152 Configuration conf, String addString) throws IOException {
153 dstFile = checkDest(srcDir.getName(), dstFS, dstFile);
154
155 if (!srcFS.isDirectory(srcDir))
156 return false;
157
158 OutputStream out = dstFS.create(dstFile);
159
160 try {
161 Path contents[] = srcFS.listPaths(srcDir);
162 for (int i = 0; i < contents.length; i++) {
163 if (srcFS.isFile(contents[i])) {
164 InputStream in = srcFS.open(contents[i]);
165 try {
166 copyContent(in, out, conf, false);
167 if (addString!=null)
168 out.write(addString.getBytes("UTF-8"));
169
170 } finally {
171 in.close();
172 }
173 }
174 }
175 } finally {
176 out.close();
177 }
178
179
180 if (deleteSource) {
181 return srcFS.delete(srcDir);
182 } else {
183 return true;
184 }
185 }
186
187 /** Copy local files to a FileSystem. */
188 public static boolean copy(File src,
189 FileSystem dstFS, Path dst,
190 boolean deleteSource,
191 Configuration conf) throws IOException {
192 dst = checkDest(src.getName(), dstFS, dst);
193
194 if (src.isDirectory()) {
195 if (!dstFS.mkdirs(dst)) {
196 return false;
197 }
198 File contents[] = src.listFiles();
199 for (int i = 0; i < contents.length; i++) {
200 copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
201 deleteSource, conf);
202 }
203 } else if (src.isFile()) {
204 InputStream in = new FileInputStream(src);
205 try {
206 copyContent(in, dstFS.create(dst), conf);
207 } finally {
208 in.close();
209 }
210 }
211 if (deleteSource) {
212 return FileUtil.fullyDelete(src);
213 } else {
214 return true;
215 }
216 }
217
218 /** Copy FileSystem files to local files. */
219 public static boolean copy(FileSystem srcFS, Path src,
220 File dst, boolean deleteSource,
221 Configuration conf) throws IOException {
222 if (srcFS.isDirectory(src)) {
223 if (!dst.mkdirs()) {
224 return false;
225 }
226 Path contents[] = srcFS.listPaths(src);
227 for (int i = 0; i < contents.length; i++) {
228 copy(srcFS, contents[i], new File(dst, contents[i].getName()),
229 deleteSource, conf);
230 }
231 } else if (srcFS.isFile(src)) {
232 InputStream in = srcFS.open(src);
233 try {
234 copyContent(in, new FileOutputStream(dst), conf);
235 } finally {
236 in.close();
237 }
238 }
239 if (deleteSource) {
240 return srcFS.delete(src);
241 } else {
242 return true;
243 }
244 }
245
246 private static void copyContent(InputStream in, OutputStream out,
247 Configuration conf) throws IOException {
248 copyContent(in, out, conf, true);
249 }
250
251
252 private static void copyContent(InputStream in, OutputStream out,
253 Configuration conf, boolean close) throws IOException {
254 byte buf[] = new byte[conf.getInt("io.file.buffer.size", 4096)];
255 try {
256 int bytesRead = in.read(buf);
257 while (bytesRead >= 0) {
258 out.write(buf, 0, bytesRead);
259 bytesRead = in.read(buf);
260 }
261 } finally {
262 if (close)
263 out.close();
264 }
265 }
266
267 private static Path checkDest(String srcName, FileSystem dstFS, Path dst)
268 throws IOException {
269 if (dstFS.exists(dst)) {
270 if (!dstFS.isDirectory(dst)) {
271 throw new IOException("Target " + dst + " already exists");
272 } else {
273 dst = new Path(dst, srcName);
274 if (dstFS.exists(dst)) {
275 throw new IOException("Target " + dst + " already exists");
276 }
277 }
278 }
279 return dst;
280 }
281
282 /**
283 * This class is only used on windows to invoke the cygpath command.
284 */
285 private static class CygPathCommand extends Command {
286 String[] command;
287 String result;
288 CygPathCommand(String path) throws IOException {
289 command = new String[]{"cygpath", "-u", path};
290 run();
291 }
292 String getResult() throws IOException {
293 return result;
294 }
295 protected String[] getExecString() {
296 return command;
297 }
298 protected void parseExecResult(BufferedReader lines) throws IOException {
299 String line = lines.readLine();
300 if (line == null) {
301 throw new IOException("Can't convert '" + command[2] +
302 " to a cygwin path");
303 }
304 result = line;
305 }
306 }
307
308 /**
309 * Convert a os-native filename to a path that works for the shell.
310 * @param filename The filename to convert
311 * @return The unix pathname
312 * @throws IOException on windows, there can be problems with the subprocess
313 */
314 public static String makeShellPath(String filename) throws IOException {
315 if (Path.WINDOWS) {
316 return new CygPathCommand(filename).getResult();
317 } else {
318 return filename;
319 }
320 }
321
322 /**
323 * Convert a os-native filename to a path that works for the shell.
324 * @param file The filename to convert
325 * @return The unix pathname
326 * @throws IOException on windows, there can be problems with the subprocess
327 */
328 public static String makeShellPath(File file) throws IOException {
329 return makeShellPath(file.toString());
330 }
331
332 /**
333 * Takes an input dir and returns the du on that local directory. Very basic
334 * implementation.
335 *
336 * @param dir
337 * The input dir to get the disk space of this local dir
338 * @return The total disk space of the input local directory
339 */
340 public static long getDU(File dir) {
341 long size = 0;
342 if (!dir.exists())
343 return 0;
344 if (!dir.isDirectory()) {
345 return dir.length();
346 } else {
347 size = dir.length();
348 File[] allFiles = dir.listFiles();
349 for (int i = 0; i < allFiles.length; i++) {
350 size = size + getDU(allFiles[i]);
351 }
352 return size;
353 }
354 }
355
356 /**
357 * Given a File input it will unzip the file in a the unzip directory
358 * passed as the second parameter
359 * @param inFile The zip file as input
360 * @param unzipDir The unzip directory where to unzip the zip file.
361 * @throws IOException
362 */
363 public static void unZip(File inFile, File unzipDir) throws IOException {
364 Enumeration entries;
365 ZipFile zipFile = new ZipFile(inFile);
366
367 try {
368 entries = zipFile.entries();
369 while (entries.hasMoreElements()) {
370 ZipEntry entry = (ZipEntry) entries.nextElement();
371 if (!entry.isDirectory()) {
372 InputStream in = zipFile.getInputStream(entry);
373 try {
374 File file = new File(unzipDir, entry.getName());
375 if (!file.getParentFile().mkdirs()) {
376 if (!file.getParentFile().isDirectory()) {
377 throw new IOException("Mkdirs failed to create " +
378 file.getParentFile().toString());
379 }
380 }
381 OutputStream out = new FileOutputStream(file);
382 try {
383 byte[] buffer = new byte[8192];
384 int i;
385 while ((i = in.read(buffer)) != -1) {
386 out.write(buffer, 0, i);
387 }
388 } finally {
389 out.close();
390 }
391 } finally {
392 in.close();
393 }
394 }
395 }
396 } finally {
397 zipFile.close();
398 }
399 }
400
401 /**
402 * Class for creating hardlinks.
403 * Supports Unix, Cygwin, WindXP.
404 *
405 */
406 public static class HardLink {
407 enum OSType {
408 OS_TYPE_UNIX,
409 OS_TYPE_WINXP;
410 }
411
412 private static String[] hardLinkCommand;
413
414 static {
415 switch(getOSType()) {
416 case OS_TYPE_WINXP:
417 hardLinkCommand = new String[] {"fsutil","hardlink","create", null, null};
418 break;
419 case OS_TYPE_UNIX:
420 default:
421 hardLinkCommand = new String[] {"ln", null, null};
422 }
423 }
424
425 static OSType getOSType() {
426 String osName = System.getProperty("os.name");
427 if (osName.indexOf("Windows") >= 0 &&
428 (osName.indexOf("XpP") >= 0 || osName.indexOf("2003") >= 0))
429 return OSType.OS_TYPE_WINXP;
430 else
431 return OSType.OS_TYPE_UNIX;
432 }
433
434 public static void createHardLink(File target,
435 File linkName) throws IOException {
436 int len = hardLinkCommand.length;
437 hardLinkCommand[len-2] = target.getCanonicalPath();
438 hardLinkCommand[len-1] = linkName.getCanonicalPath();
439 // execute shell command
440 Process process = Runtime.getRuntime().exec(hardLinkCommand);
441 try {
442 if (process.waitFor() != 0) {
443 String errMsg = new BufferedReader(new InputStreamReader(
444 process.getInputStream())).readLine();
445 if (errMsg == null) errMsg = "";
446 String inpMsg = new BufferedReader(new InputStreamReader(
447 process.getErrorStream())).readLine();
448 if (inpMsg == null) inpMsg = "";
449 throw new IOException(errMsg + inpMsg);
450 }
451 } catch (InterruptedException e) {
452 throw new IOException(StringUtils.stringifyException(e));
453 } finally {
454 process.destroy();
455 }
456 }
457 }
458
459 /**
460 * Create a soft link between a src and destination
461 * only on a local disk. HDFS does not support this
462 * @param target the target for symlink
463 * @param linkname the symlink
464 * @return value returned by the command
465 */
466 public static int symLink(String target, String linkname) throws IOException{
467 String cmd = "ln -s " + target + " " + linkname;
468 Process p = Runtime.getRuntime().exec(cmd, null);
469 int returnVal = -1;
470 try{
471 returnVal = p.waitFor();
472 } catch(InterruptedException e){
473 //do nothing as of yet
474 }
475 return returnVal;
476 }
477
478 /**
479 * Change the permissions on a filename.
480 * @param filename the name of the file to change
481 * @param perm the permission string
482 * @return the exit code from the command
483 * @throws IOException
484 * @throws InterruptedException
485 */
486 public static int chmod(String filename, String perm
487 ) throws IOException, InterruptedException {
488 String cmd = "chmod " + perm + " " + filename;
489 Process p = Runtime.getRuntime().exec(cmd, null);
490 return p.waitFor();
491 }
492
493 /**
494 * Create a tmp file for a base file.
495 * @param basefile the base file of the tmp
496 * @param prefix file name prefix of tmp
497 * @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
498 * @return a newly created tmp file
499 * @exception IOException If a tmp file cannot created
500 * @see java.io.File#createTempFile(String, String, File)
501 * @see java.io.File#deleteOnExit()
502 */
503 public static final File createLocalTempFile(final File basefile,
504 final String prefix,
505 final boolean isDeleteOnExit)
506 throws IOException {
507 File tmp = File.createTempFile(prefix + basefile.getName(),
508 "", basefile.getParentFile());
509 if (isDeleteOnExit) {
510 tmp.deleteOnExit();
511 }
512 return tmp;
513 }
514
515 //XXX These functions should be in IO Utils rather than FileUtil
516 // Reads len bytes in a loop.
517 public static void readFully( InputStream in, byte buf[],
518 int off, int len ) throws IOException {
519 int toRead = len;
520 while ( toRead > 0 ) {
521 int ret = in.read( buf, off, toRead );
522 if ( ret < 0 ) {
523 throw new IOException( "Premeture EOF from inputStream");
524 }
525 toRead -= ret;
526 off += ret;
527 }
528 }
529
530 public static void skipFully( InputStream in, long len ) throws IOException {
531 long toSkip = len;
532 while ( toSkip > 0 ) {
533 long ret = in.skip( toSkip );
534 if ( ret < 0 ) {
535 throw new IOException( "Premeture EOF from inputStream");
536 }
537 toSkip -= ret;
538 }
539 }
540
541 public static void closeSocket( Socket sock ) {
542 // avoids try { close() } dance
543 if ( sock != null ) {
544 try {
545 sock.close();
546 } catch ( IOException ignored ) {
547 }
548 }
549 }
550
551 public static void closeStream(Closeable closeable ) {
552 // avoids try { close() } dance
553 if ( closeable != null ) {
554 try {
555 closeable.close();
556 } catch ( IOException ignored ) {
557 }
558 }
559 }
560 }
Save This Page