1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.hadoop.fs;
19
20 import java.io;
21 import java.net;
22 import java.util;
23 import java.util.regex.Pattern;
24
25 import org.apache.commons.logging;
26
27 import org.apache.hadoop.dfs;
28 import org.apache.hadoop.conf;
29 import org.apache.hadoop.util;
30
31 /****************************************************************
32 * An abstract base class for a fairly generic filesystem. It
33 * may be implemented as a distributed filesystem, or as a "local"
34 * one that reflects the locally-connected disk. The local version
35 * exists for small Hadopp instances and for testing.
36 *
37 * <p>
38 *
39 * All user code that may potentially use the Hadoop Distributed
40 * File System should be written to use a FileSystem object. The
41 * Hadoop DFS is a multi-machine system that appears as a single
42 * disk. It's useful because of its fault tolerance and potentially
43 * very large capacity.
44 *
45 * <p>
46 * The local implementation is {@link LocalFileSystem} and distributed
47 * implementation is {@link DistributedFileSystem}.
48 *****************************************************************/
49 public abstract class FileSystem extends Configured {
50 public static final Log LOG = LogFactory.getLog("org.apache.hadoop.fs.FileSystem");
51
52 // cache indexed by URI scheme and authority
53 private static final Map<String,Map<String,FileSystem>> CACHE
54 = new HashMap<String,Map<String,FileSystem>>();
55 /**
56 * Parse the cmd-line args, starting at i. Remove consumed args
57 * from array. We expect param in the form:
58 * '-local | -dfs <namenode:port>'
59 */
60 public static FileSystem parseArgs(String argv[], int i, Configuration conf) throws IOException {
61 /**
62 if (argv.length - i < 1) {
63 throw new IOException("Must indicate filesystem type for DFS");
64 }
65 */
66 int orig = i;
67 FileSystem fs = null;
68 String cmd = argv[i];
69 if ("-dfs".equals(cmd)) {
70 i++;
71 InetSocketAddress addr = DataNode.createSocketAddr(argv[i++]);
72 fs = new DistributedFileSystem(addr, conf);
73 } else if ("-local".equals(cmd)) {
74 i++;
75 fs = FileSystem.getLocal(conf);
76 } else {
77 fs = get(conf); // using default
78 LOG.info("No FS indicated, using default:"+fs.getName());
79
80 }
81 System.arraycopy(argv, i, argv, orig, argv.length - i);
82 for (int j = argv.length - i; j < argv.length; j++) {
83 argv[j] = null;
84 }
85 return fs;
86 }
87
88 /** Returns the configured filesystem implementation.*/
89 public static FileSystem get(Configuration conf) throws IOException {
90 return getNamed(conf.get("fs.default.name", "file:///"), conf);
91 }
92
93 /** Called after a new FileSystem instance is constructed.
94 * @param name a uri whose authority section names the host, port, etc.
95 * for this FileSystem
96 * @param conf the configuration
97 */
98 public abstract void initialize(URI name, Configuration conf)
99 throws IOException;
100
101 /** Returns a URI whose scheme and authority identify this FileSystem.*/
102 public abstract URI getUri();
103
104 /** @deprecated call #getUri() instead.*/
105 public String getName() { return getUri().toString(); }
106
107 /** @deprecated call #get(URI,Configuration) instead. */
108 public static FileSystem getNamed(String name, Configuration conf)
109 throws IOException {
110
111 // convert old-format name to new-format name
112 if (name.equals("local")) { // "local" is now "file:///".
113 name = "file:///";
114 } else if (name.indexOf('/')==-1) { // unqualified is "hdfs://"
115 name = "hdfs://"+name;
116 }
117
118 return get(URI.create(name), conf);
119 }
120
121 /**
122 * Get the local file syste
123 * @param conf the configuration to configure the file system with
124 * @return a LocalFileSystem
125 */
126 public static LocalFileSystem getLocal(Configuration conf)
127 throws IOException {
128 return (LocalFileSystem)get(LocalFileSystem.NAME, conf);
129 }
130
131 /** Returns the FileSystem for this URI's scheme and authority. The scheme
132 * of the URI determines a configuration property name,
133 * <tt>fs.<i>scheme</i>.class</tt> whose value names the FileSystem class.
134 * The entire URI is passed to the FileSystem instance's initialize method.
135 */
136 public static synchronized FileSystem get(URI uri, Configuration conf)
137 throws IOException {
138
139 String scheme = uri.getScheme();
140 String authority = uri.getAuthority();
141
142 if (scheme == null) { // no scheme: use default FS
143 return get(conf);
144 }
145
146 Map<String,FileSystem> authorityToFs = CACHE.get(scheme);
147 if (authorityToFs == null) {
148 authorityToFs = new HashMap<String,FileSystem>();
149 CACHE.put(scheme, authorityToFs);
150 }
151
152 FileSystem fs = authorityToFs.get(authority);
153 if (fs == null) {
154 Class fsClass = conf.getClass("fs."+scheme+".impl", null);
155 if (fsClass == null) {
156 throw new IOException("No FileSystem for scheme: " + scheme);
157 }
158 fs = (FileSystem)ReflectionUtils.newInstance(fsClass, conf);
159 fs.initialize(uri, conf);
160 authorityToFs.put(authority, fs);
161 }
162
163 return fs;
164 }
165
166 /**
167 * Close all cached filesystems. Be sure those filesystems are not
168 * used anymore.
169 *
170 * @throws IOException
171 */
172 public static synchronized void closeAll() throws IOException{
173 for(Map<String, FileSystem> fss : CACHE.values()){
174 for(FileSystem fs : fss.values()){
175 fs.close();
176 }
177 }
178 }
179
180 /** Make sure that a path specifies a FileSystem. */
181 public Path makeQualified(Path path) {
182 checkPath(path);
183 return path.makeQualified(this);
184 }
185
186 ///////////////////////////////////////////////////////////////
187 // FileSystem
188 ///////////////////////////////////////////////////////////////
189
190 protected FileSystem() {
191 super(null);
192 }
193
194 /** Check that a Path belongs to this FileSystem. */
195 protected void checkPath(Path path) {
196 URI uri = path.toUri();
197 if (uri.getScheme() == null) // fs is relative
198 return;
199 String thisAuthority = this.getUri().getAuthority();
200 String thatAuthority = uri.getAuthority();
201 if (!(this.getUri().getScheme().equals(uri.getScheme()) &&
202 (thisAuthority == thatAuthority ||
203 (thisAuthority != null && thisAuthority.equals(thatAuthority))))){
204 throw new IllegalArgumentException("Wrong FS: "+path+
205 ", expected: "+this.getUri());
206 }
207 }
208
209 /**
210 * Return a 2D array of size 1x1 or greater, containing hostnames
211 * where portions of the given file can be found. For a nonexistent
212 * file or regions, null will be returned.
213 *
214 * This call is most helpful with DFS, where it returns
215 * hostnames of machines that contain the given file.
216 *
217 * The FileSystem will simply return an elt containing 'localhost'.
218 */
219 public String[][] getFileCacheHints(Path f, long start, long len)
220 throws IOException {
221 if (!exists(f)) {
222 return null;
223 } else {
224 String result[][] = new String[1][];
225 result[0] = new String[1];
226 result[0][0] = "localhost";
227 return result;
228 }
229 }
230
231 /**
232 * Opens an FSDataInputStream at the indicated Path.
233 * @param f the file name to open
234 * @param bufferSize the size of the buffer to be used.
235 */
236 public abstract FSDataInputStream open(Path f, int bufferSize)
237 throws IOException;
238
239 /**
240 * Opens an FSDataInputStream at the indicated Path.
241 * @param f the file to open
242 */
243 public FSDataInputStream open(Path f) throws IOException {
244 return open(f, getConf().getInt("io.file.buffer.size", 4096));
245 }
246
247 /**
248 * Opens an FSDataOutputStream at the indicated Path.
249 * Files are overwritten by default.
250 */
251 public FSDataOutputStream create(Path f) throws IOException {
252 return create(f, true,
253 getConf().getInt("io.file.buffer.size", 4096),
254 getDefaultReplication(),
255 getDefaultBlockSize());
256 }
257
258 /**
259 * Create an FSDataOutputStream at the indicated Path with write-progress
260 * reporting.
261 * Files are overwritten by default.
262 */
263 public FSDataOutputStream create(Path f, Progressable progress) throws IOException {
264 return create(f, true,
265 getConf().getInt("io.file.buffer.size", 4096),
266 getDefaultReplication(),
267 getDefaultBlockSize(), progress);
268 }
269
270 /**
271 * Opens an FSDataOutputStream at the indicated Path.
272 * Files are overwritten by default.
273 */
274 public FSDataOutputStream create(Path f, short replication)
275 throws IOException {
276 return create(f, true,
277 getConf().getInt("io.file.buffer.size", 4096),
278 replication,
279 getDefaultBlockSize());
280 }
281
282 /**
283 * Opens an FSDataOutputStream at the indicated Path with write-progress
284 * reporting.
285 * Files are overwritten by default.
286 */
287 public FSDataOutputStream create(Path f, short replication, Progressable progress)
288 throws IOException {
289 return create(f, true,
290 getConf().getInt("io.file.buffer.size", 4096),
291 replication,
292 getDefaultBlockSize(), progress);
293 }
294
295
296 /**
297 * Opens an FSDataOutputStream at the indicated Path.
298 * @param f the file name to open
299 * @param overwrite if a file with this name already exists, then if true,
300 * the file will be overwritten, and if false an error will be thrown.
301 * @param bufferSize the size of the buffer to be used.
302 */
303 public FSDataOutputStream create(Path f,
304 boolean overwrite,
305 int bufferSize
306 ) throws IOException {
307 return create(f, overwrite, bufferSize,
308 getDefaultReplication(),
309 getDefaultBlockSize());
310 }
311
312 /**
313 * Opens an FSDataOutputStream at the indicated Path with write-progress
314 * reporting.
315 * @param f the file name to open
316 * @param overwrite if a file with this name already exists, then if true,
317 * the file will be overwritten, and if false an error will be thrown.
318 * @param bufferSize the size of the buffer to be used.
319 */
320 public FSDataOutputStream create(Path f,
321 boolean overwrite,
322 int bufferSize,
323 Progressable progress
324 ) throws IOException {
325 return create(f, overwrite, bufferSize,
326 getDefaultReplication(),
327 getDefaultBlockSize(), progress);
328 }
329
330
331 /**
332 * Opens an FSDataOutputStream at the indicated Path.
333 * @param f the file name to open
334 * @param overwrite if a file with this name already exists, then if true,
335 * the file will be overwritten, and if false an error will be thrown.
336 * @param bufferSize the size of the buffer to be used.
337 * @param replication required block replication for the file.
338 */
339 public FSDataOutputStream create(Path f,
340 boolean overwrite,
341 int bufferSize,
342 short replication,
343 long blockSize
344 ) throws IOException {
345 return create(f, overwrite, bufferSize, replication, blockSize, null);
346 }
347
348 /**
349 * Opens an FSDataOutputStream at the indicated Path with write-progress
350 * reporting.
351 * @param f the file name to open
352 * @param overwrite if a file with this name already exists, then if true,
353 * the file will be overwritten, and if false an error will be thrown.
354 * @param bufferSize the size of the buffer to be used.
355 * @param replication required block replication for the file.
356 */
357 public abstract FSDataOutputStream create(Path f,
358 boolean overwrite,
359 int bufferSize,
360 short replication,
361 long blockSize,
362 Progressable progress
363 ) throws IOException;
364
365 /**
366 * Creates the given Path as a brand-new zero-length file. If
367 * create fails, or if it already existed, return false.
368 */
369 public boolean createNewFile(Path f) throws IOException {
370 if (exists(f)) {
371 return false;
372 } else {
373 create(f, false, getConf().getInt("io.file.buffer.size", 4096)).close();
374 return true;
375 }
376 }
377
378 /**
379 * Get replication.
380 *
381 * @deprecated Use getFileStatus() instead
382 * @param src file name
383 * @return file replication
384 * @throws IOException
385 */
386 @Deprecated
387 public short getReplication(Path src) throws IOException {
388 return getFileStatus(src).getReplication();
389 }
390
391 /**
392 * Set replication for an existing file.
393 *
394 * @param src file name
395 * @param replication new replication
396 * @throws IOException
397 * @return true if successful;
398 * false if file does not exist or is a directory
399 */
400 public boolean setReplication(Path src, short replication)
401 throws IOException {
402 return true;
403 }
404
405 /**
406 * Renames Path src to Path dst. Can take place on local fs
407 * or remote DFS.
408 */
409 public abstract boolean rename(Path src, Path dst) throws IOException;
410
411 /** Delete a file */
412 public abstract boolean delete(Path f) throws IOException;
413
414 /** Check if exists.
415 * @param f source file
416 */
417 public abstract boolean exists(Path f) throws IOException;
418
419 /** True iff the named path is a directory. */
420 /** @deprecated Use getFileStatus() instead */ @Deprecated
421 public boolean isDirectory(Path f) throws IOException {
422 try {
423 return getFileStatus(f).isDir();
424 } catch (IOException e) {
425 return false; // f does not exist
426 }
427 }
428
429 /** True iff the named path is a regular file. */
430 public boolean isFile(Path f) throws IOException {
431 if (exists(f) && !isDirectory(f)) {
432 return true;
433 } else {
434 return false;
435 }
436 }
437
438 /** The number of bytes in a file. */
439 /** @deprecated Use getFileStatus() instead */ @Deprecated
440 public long getLength(Path f) throws IOException {
441 return getFileStatus(f).getLen();
442 }
443
444 /** Return the number of bytes of the given path
445 * If <i>f</i> is a file, return the size of the file;
446 * If <i>f</i> is a directory, return the size of the directory tree
447 */
448 public long getContentLength(Path f) throws IOException {
449 if (!isDirectory(f)) {
450 // f is a file
451 return getLength(f);
452 }
453
454 // f is a diretory
455 Path[] contents = listPaths(f);
456 long size = 0;
457 for(int i=0; i<contents.length; i++) {
458 size += getContentLength(contents[i]);
459 }
460 return size;
461 }
462
463 final private static PathFilter DEFAULT_FILTER = new PathFilter() {
464 public boolean accept(Path file) {
465 return true;
466 }
467 };
468
469 /** List files in a directory. */
470 public abstract Path[] listPaths(Path f) throws IOException;
471
472 /**
473 * Filter files in the given pathes using the default checksum filter.
474 * @param files a list of paths
475 * @return a list of files under the source paths
476 * @exception IOException
477 */
478 public Path[] listPaths(Path[] files) throws IOException {
479 return listPaths(files, DEFAULT_FILTER);
480 }
481
482 /** Filter files in a directory. */
483 private void listPaths(ArrayList<Path> results, Path f, PathFilter filter)
484 throws IOException {
485 Path listing[] = listPaths(f);
486 if (listing != null) {
487 for (int i = 0; i < listing.length; i++) {
488 if (filter.accept(listing[i])) {
489 results.add(listing[i]);
490 }
491 }
492 }
493 }
494
495 /** Filter files in a directory. */
496 public Path[] listPaths(Path f, PathFilter filter) throws IOException {
497 ArrayList<Path> results = new ArrayList<Path>();
498 listPaths(results, f, filter);
499 return (Path[]) results.toArray(new Path[results.size()]);
500 }
501
502 /**
503 * Filter files in a list directories using user-supplied path filter.
504 * @param files a list of paths
505 * @return a list of files under the source paths
506 * @exception IOException
507 */
508 public Path[] listPaths(Path[] files, PathFilter filter)
509 throws IOException {
510 ArrayList<Path> results = new ArrayList<Path>();
511 for(int i=0; i<files.length; i++) {
512 listPaths(results, files[i], filter);
513 }
514 return (Path[]) results.toArray(new Path[results.size()]);
515 }
516
517 /**
518 * <p>Return all the files that match filePattern and are not checksum
519 * files. Results are sorted by their names.
520 *
521 * <p>
522 * A filename pattern is composed of <i>regular</i> characters and
523 * <i>special pattern matching</i> characters, which are:
524 *
525 * <dl>
526 * <dd>
527 * <dl>
528 * <p>
529 * <dt> <tt> ? </tt>
530 * <dd> Matches any single character.
531 *
532 * <p>
533 * <dt> <tt> * </tt>
534 * <dd> Matches zero or more characters.
535 *
536 * <p>
537 * <dt> <tt> [<i>abc</i>] </tt>
538 * <dd> Matches a single character from character set
539 * <tt>{<i>a,b,c</i>}</tt>.
540 *
541 * <p>
542 * <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
543 * <dd> Matches a single character from the character range
544 * <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be
545 * lexicographically less than or equal to character <tt><i>b</i></tt>.
546 *
547 * <p>
548 * <dt> <tt> [^<i>a</i>] </tt>
549 * <dd> Matches a single character that is not from character set or range
550 * <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur
551 * immediately to the right of the opening bracket.
552 *
553 * <p>
554 * <dt> <tt> \<i>c</i> </tt>
555 * <dd> Removes (escapes) any special meaning of character <i>c</i>.
556 *
557 * </dl>
558 * </dd>
559 * </dl>
560 *
561 * @param filePattern a regular expression specifying file pattern
562
563 * @return an array of paths that match the file pattern
564 * @throws IOException
565 */
566 public Path[] globPaths(Path filePattern) throws IOException {
567 return globPaths(filePattern, DEFAULT_FILTER);
568 }
569
570 /** glob all the file names that matches filePattern
571 * and is accepted by filter.
572 */
573 public Path[] globPaths(Path filePattern, PathFilter filter)
574 throws IOException {
575 Path [] parents = new Path[1];
576 int level = 0;
577 String filename = filePattern.toUri().getPath();
578 if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
579 parents[0] = filePattern;
580 return parents;
581 }
582
583 String [] components = filename.split(Path.SEPARATOR);
584 if (filePattern.isAbsolute()) {
585 parents[0] = new Path(Path.SEPARATOR);
586 level = 1;
587 } else {
588 parents[0] = new Path(Path.CUR_DIR);
589 }
590
591 Path[] results = globPathsLevel(parents, components, level, filter);
592 Arrays.sort(results);
593 return results;
594 }
595
596 private Path[] globPathsLevel(Path[] parents,
597 String [] filePattern, int level, PathFilter filter) throws IOException {
598 if (level == filePattern.length)
599 return parents;
600 GlobFilter fp = new GlobFilter(filePattern[level], filter);
601 if (fp.hasPattern()) {
602 parents = listPaths(parents, fp);
603 } else {
604 for(int i=0; i<parents.length; i++) {
605 parents[i] = new Path(parents[i], filePattern[level]);
606 }
607 }
608 return globPathsLevel(parents, filePattern, level+1, filter);
609 }
610
611 private static class GlobFilter implements PathFilter {
612 private PathFilter userFilter = DEFAULT_FILTER;
613 private Pattern regex;
614 private boolean hasPattern = false;
615
616 /** Default pattern character: Escape any special meaning. */
617 private static final char PAT_ESCAPE = '\\';
618 /** Default pattern character: Any single character. */
619 private static final char PAT_ANY = '.';
620 /** Default pattern character: Character set close. */
621 private static final char PAT_SET_CLOSE = ']';
622
623 GlobFilter() {
624 }
625
626 GlobFilter(String filePattern) throws IOException {
627 setRegex(filePattern);
628 }
629
630 GlobFilter(String filePattern, PathFilter filter) throws IOException {
631 userFilter = filter;
632 setRegex(filePattern);
633 }
634
635 void setRegex(String filePattern) throws IOException {
636 int len;
637 int setOpen;
638 boolean setRange;
639 StringBuffer fileRegex = new StringBuffer();
640
641 // Validate the pattern
642 len = filePattern.length();
643 if (len == 0)
644 return;
645
646 setOpen = 0;
647 setRange = false;
648
649 for (int i = 0; i < len; i++) {
650 char pCh;
651
652 // Examine a single pattern character
653 pCh = filePattern.charAt(i);
654 if (pCh == PAT_ESCAPE) {
655 fileRegex.append(pCh);
656 i++;
657 if (i >= len)
658 error("An escaped character does not present", filePattern, i);
659 pCh = filePattern.charAt(i);
660 } else if (pCh == '.') {
661 fileRegex.append(PAT_ESCAPE);
662 } else if (pCh == '*') {
663 fileRegex.append(PAT_ANY);
664 hasPattern = true;
665 } else if (pCh == '?') {
666 pCh = PAT_ANY;
667 hasPattern = true;
668 } else if (pCh == '[' && setOpen == 0) {
669 setOpen++;
670 hasPattern = true;
671 } else if (pCh == '^' && setOpen > 0) {
672 } else if (pCh == '-' && setOpen > 0) {
673 // Character set range
674 setRange = true;
675 } else if (pCh == PAT_SET_CLOSE && setRange) {
676 // Incomplete character set range
677 error("Incomplete character set range", filePattern, i);
678 } else if (pCh == PAT_SET_CLOSE && setOpen > 0) {
679 // End of a character set
680 if (setOpen < 2)
681 error("Unexpected end of set", filePattern, i);
682 setOpen = 0;
683 } else if (setOpen > 0) {
684 // Normal character, or the end of a character set range
685 setOpen++;
686 setRange = false;
687 }
688 fileRegex.append(pCh);
689 }
690
691 // Check for a well-formed pattern
692 if (setOpen > 0 || setRange) {
693 // Incomplete character set or character range
694 error("Expecting set closure character or end of range", filePattern,
695 len);
696 }
697 regex = Pattern.compile(fileRegex.toString());
698 }
699
700 boolean hasPattern() {
701 return hasPattern;
702 }
703
704 public boolean accept(Path path) {
705 return regex.matcher(path.getName()).matches() && userFilter.accept(path);
706 }
707
708 private void error(String s, String pattern, int pos) throws IOException {
709 throw new IOException("Illegal file pattern: "
710 +s+ " for glob "+ pattern + " at " + pos);
711 }
712 }
713
714 /**
715 * Set the current working directory for the given file system. All relative
716 * paths will be resolved relative to it.
717 *
718 * @param new_dir
719 */
720 public abstract void setWorkingDirectory(Path new_dir);
721
722 /**
723 * Get the current working directory for the given file system
724 * @return the directory pathname
725 */
726 public abstract Path getWorkingDirectory();
727
728 /**
729 * Make the given file and all non-existent parents into
730 * directories. Has the semantics of Unix 'mkdir -p'.
731 * Existence of the directory hierarchy is not an error.
732 */
733 public abstract boolean mkdirs(Path f) throws IOException;
734
735 /**
736 * Obtain a lock on the given Path
737 *
738 * @deprecated FS does not support file locks anymore.
739 */
740 @Deprecated
741 public void lock(Path f, boolean shared) throws IOException {}
742
743 /**
744 * Release the lock
745 *
746 * @deprecated FS does not support file locks anymore.
747 */
748 @Deprecated
749 public void release(Path f) throws IOException {}
750
751 /**
752 * The src file is on the local disk. Add it to FS at
753 * the given dst name and the source is kept intact afterwards
754 */
755 public void copyFromLocalFile(Path src, Path dst)
756 throws IOException {
757 copyFromLocalFile(false, src, dst);
758 }
759
760 /**
761 * The src file is on the local disk. Add it to FS at
762 * the given dst name, removing the source afterwards.
763 */
764 public void moveFromLocalFile(Path src, Path dst)
765 throws IOException {
766 copyFromLocalFile(true, src, dst);
767 }
768
769 /**
770 * The src file is on the local disk. Add it to FS at
771 * the given dst name.
772 * delSrc indicates if the source should be removed
773 */
774 public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
775 throws IOException {
776 FileUtil.copy(getLocal(getConf()), src, this, dst, delSrc, getConf());
777 }
778
779 /**
780 * The src file is under FS, and the dst is on the local disk.
781 * Copy it from FS control to the local dst name.
782 */
783 public void copyToLocalFile(Path src, Path dst) throws IOException {
784 copyToLocalFile(false, src, dst);
785 }
786
787 /**
788 * The src file is under FS, and the dst is on the local disk.
789 * Copy it from FS control to the local dst name.
790 * Remove the source afterwards
791 */
792 public void moveToLocalFile(Path src, Path dst) throws IOException {
793 copyToLocalFile(true, src, dst);
794 }
795
796 /**
797 * The src file is under FS, and the dst is on the local disk.
798 * Copy it from FS control to the local dst name.
799 * delSrc indicates if the src will be removed or not.
800 */
801 public void copyToLocalFile(boolean delSrc, Path src, Path dst)
802 throws IOException {
803 FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc, getConf());
804 }
805
806 /**
807 * Returns a local File that the user can write output to. The caller
808 * provides both the eventual FS target name and the local working
809 * file. If the FS is local, we write directly into the target. If
810 * the FS is remote, we write into the tmp local area.
811 */
812 public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
813 throws IOException {
814 return tmpLocalFile;
815 }
816
817 /**
818 * Called when we're all done writing to the target. A local FS will
819 * do nothing, because we've written to exactly the right place. A remote
820 * FS will copy the contents of tmpLocalFile to the correct target at
821 * fsOutputFile.
822 */
823 public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
824 throws IOException {
825 moveFromLocalFile(tmpLocalFile, fsOutputFile);
826 }
827
828 /**
829 * No more filesystem operations are needed. Will
830 * release any held locks.
831 */
832 public void close() throws IOException {
833 URI uri = getUri();
834 synchronized (FileSystem.class) {
835 Map<String,FileSystem> authorityToFs = CACHE.get(uri.getScheme());
836 if (authorityToFs != null) {
837 authorityToFs.remove(uri.getAuthority());
838 }
839 }
840 }
841
842 /** Return the total size of all files in the filesystem.*/
843 public long getUsed() throws IOException{
844 long used = 0;
845 Path[] files = listPaths(new Path("/"));
846 for(Path file:files){
847 used += getContentLength(file);
848 }
849 return used;
850 }
851
852 /**
853 * Get the block size for a particular file.
854 * @param f the filename
855 * @return the number of bytes in a block
856 */
857 /** @deprecated Use getFileStatus() instead */ @Deprecated
858 public long getBlockSize(Path f) throws IOException {
859 return getFileStatus(f).getBlockSize();
860 }
861
862 /** Return the number of bytes that large input files should be optimally
863 * be split into to minimize i/o time. */
864 public long getDefaultBlockSize() {
865 // default to 32MB: large enough to minimize the impact of seeks
866 return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024);
867 }
868
869 /**
870 * Get the default replication.
871 */
872 public short getDefaultReplication() { return 1; }
873
874 /*
875 * Return a file status object that represents the
876 * file.
877 * @param f The path to the file we want information from
878 * @return filestatus object
879 * @throws IOException see specific implementation
880 */
881 public abstract FileStatus getFileStatus(Path f) throws IOException;
882 }
Save This Page