hadoop Ls 源码
haddop Ls 代码
文件路径：/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.shell;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedList;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.ContentSummary;
/**
* Get a listing of all files in that match the file patterns.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
class Ls extends FsCommand {
public static void registerCommands(CommandFactory factory) {
factory.addClass(Ls.class, "-ls");
factory.addClass(Lsr.class, "-lsr");
}
private static final String OPTION_PATHONLY = "C";
private static final String OPTION_DIRECTORY = "d";
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HIDENONPRINTABLE = "q";
private static final String OPTION_RECURSIVE = "R";
private static final String OPTION_REVERSE = "r";
private static final String OPTION_MTIME = "t";
private static final String OPTION_ATIME = "u";
private static final String OPTION_SIZE = "S";
private static final String OPTION_ECPOLICY = "e";
public static final String NAME = "ls";
public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-" +
OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" +
OPTION_HIDENONPRINTABLE + "] [-" + OPTION_RECURSIVE + "] [-" +
OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] [-" +
OPTION_ATIME + "] [-" + OPTION_ECPOLICY +"] [<path> ...]";
public static final String DESCRIPTION =
"List the contents that match the specified file pattern. If " +
"path is not specified, the contents of /user/<currentUser> " +
"will be listed. For a directory a list of its direct children " +
"is returned (unless -" + OPTION_DIRECTORY +
" option is specified).\n\n" +
"Directory entries are of the form:\n" +
"\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
"and file entries are of the form:\n" +
"\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" +
" -" + OPTION_PATHONLY +
" Display the paths of files and directories only.\n" +
" -" + OPTION_DIRECTORY +
" Directories are listed as plain files.\n" +
" -" + OPTION_HUMAN +
" Formats the sizes of files in a human-readable fashion\n" +
" rather than a number of bytes.\n" +
" -" + OPTION_HIDENONPRINTABLE +
" Print ? instead of non-printable characters.\n" +
" -" + OPTION_RECURSIVE +
" Recursively list the contents of directories.\n" +
" -" + OPTION_MTIME +
" Sort files by modification time (most recent first).\n" +
" -" + OPTION_SIZE +
" Sort files by size.\n" +
" -" + OPTION_REVERSE +
" Reverse the order of the sort.\n" +
" -" + OPTION_ATIME +
" Use time of last access instead of modification for\n" +
" display and sorting.\n"+
" -" + OPTION_ECPOLICY +
" Display the erasure coding policy of files and directories.\n";
protected final SimpleDateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
protected String lineFormat;
private boolean pathOnly;
protected boolean dirRecurse;
private boolean orderReverse;
private boolean orderTime;
private boolean orderSize;
private boolean useAtime;
private boolean displayECPolicy;
private Comparator<PathData> orderComparator;
protected boolean humanReadable = false;
/** Whether to print ? instead of non-printable characters. */
private boolean hideNonPrintable = false;
protected Ls() {}
protected Ls(Configuration conf) {
super(conf);
}
protected String formatSize(long size) {
return humanReadable
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
: String.valueOf(size);
}
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
OPTION_PATHONLY, OPTION_DIRECTORY, OPTION_HUMAN,
OPTION_HIDENONPRINTABLE, OPTION_RECURSIVE, OPTION_REVERSE,
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME, OPTION_ECPOLICY);
cf.parse(args);
pathOnly = cf.getOpt(OPTION_PATHONLY);
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
humanReadable = cf.getOpt(OPTION_HUMAN);
hideNonPrintable = cf.getOpt(OPTION_HIDENONPRINTABLE);
orderReverse = cf.getOpt(OPTION_REVERSE);
orderTime = cf.getOpt(OPTION_MTIME);
orderSize = !orderTime && cf.getOpt(OPTION_SIZE);
useAtime = cf.getOpt(OPTION_ATIME);
displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
if (args.isEmpty()) args.add(Path.CUR_DIR);
initialiseOrderComparator();
}
/**
* Should display only paths of files and directories.
* @return true display paths only, false display all fields
*/
@VisibleForTesting
boolean isPathOnly() {
return this.pathOnly;
}
/**
* Should the contents of the directory be shown or just the directory?
* @return true if directory contents, false if just directory
*/
@VisibleForTesting
boolean isDirRecurse() {
return this.dirRecurse;
}
/**
* Should file sizes be returned in human readable format rather than bytes?
* @return true is human readable, false if bytes
*/
@VisibleForTesting
boolean isHumanReadable() {
return this.humanReadable;
}
@VisibleForTesting
private boolean isHideNonPrintable() {
return hideNonPrintable;
}
/**
* Should directory contents be displayed in reverse order
* @return true reverse order, false default order
*/
@VisibleForTesting
boolean isOrderReverse() {
return this.orderReverse;
}
/**
* Should directory contents be displayed in mtime order.
* @return true mtime order, false default order
*/
@VisibleForTesting
boolean isOrderTime() {
return this.orderTime;
}
/**
* Should directory contents be displayed in size order.
* @return true size order, false default order
*/
@VisibleForTesting
boolean isOrderSize() {
return this.orderSize;
}
/**
* Should access time be used rather than modification time.
* @return true use access time, false use modification time
*/
@VisibleForTesting
boolean isUseAtime() {
return this.useAtime;
}
/**
* Should EC policies be displayed.
* @return true display EC policies, false doesn't display EC policies
*/
@VisibleForTesting
boolean isDisplayECPolicy() {
return this.displayECPolicy;
}
@Override
protected void processPathArgument(PathData item) throws IOException {
if (isDisplayECPolicy() && item.fs.getContentSummary(item.path)
.getErasureCodingPolicy() == null) {
throw new UnsupportedOperationException("FileSystem "
+ item.fs.getUri() + " does not support Erasure Coding");
}
// implicitly recurse once for cmdline directories
if (dirRecurse && item.stat.isDirectory()) {
recursePath(item);
} else {
super.processPathArgument(item);
}
}
@Override
protected boolean isSorted() {
// use the non-iterative method for listing because explicit sorting is
// required based on time/size/reverse or Total number of entries
// required to print summary first when non-recursive.
return !isRecursive() || isOrderTime() || isOrderSize() || isOrderReverse();
}
@Override
protected int getListingGroupSize() {
if (pathOnly) {
// If there is a need of printing only paths, then no grouping required
return 0;
}
/*
* LS output should be formatted properly. Grouping 100 items and formatting
* the output to reduce the creation of huge sized arrays. This method will
* be called only when recursive is set.
*/
return 100;
}
@Override
protected void processPaths(PathData parent, PathData... items)
throws IOException {
if (parent != null && !isRecursive() && items.length != 0) {
if (!pathOnly) {
out.println("Found " + items.length + " items");
}
Arrays.sort(items, getOrderComparator());
}
if (!pathOnly) {
adjustColumnWidths(items);
}
super.processPaths(parent, items);
}
@Override
protected void processPath(PathData item) throws IOException {
if (pathOnly) {
out.println(item.toString());
return;
}
FileStatus stat = item.stat;
if (displayECPolicy) {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),
stat.getPermission() + (stat.hasAcl() ? "+" : " "),
(stat.isFile() ? stat.getReplication() : "-"),
stat.getOwner(),
stat.getGroup(),
contentSummary.getErasureCodingPolicy(),
formatSize(stat.getLen()),
dateFormat.format(new Date(isUseAtime()
? stat.getAccessTime()
: stat.getModificationTime())),
isHideNonPrintable() ? new PrintableString(item.toString()) : item);
out.println(line);
} else {
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),
stat.getPermission() + (stat.hasAcl() ? "+" : " "),
(stat.isFile() ? stat.getReplication() : "-"),
stat.getOwner(),
stat.getGroup(),
formatSize(stat.getLen()),
dateFormat.format(new Date(isUseAtime()
? stat.getAccessTime()
: stat.getModificationTime())),
isHideNonPrintable() ? new PrintableString(item.toString()) : item);
out.println(line);
}
}
/**
* Compute column widths and rebuild the format string
* @param items to find the max field width for each column
*/
private void adjustColumnWidths(PathData items[]) throws IOException {
for (PathData item : items) {
FileStatus stat = item.stat;
maxRepl = maxLength(maxRepl, stat.getReplication());
maxLen = maxLength(maxLen, stat.getLen());
maxOwner = maxLength(maxOwner, stat.getOwner());
maxGroup = maxLength(maxGroup, stat.getGroup());
}
StringBuilder fmt = new StringBuilder();
fmt.append("%s%s") // permission string
.append("%" + maxRepl + "s ")
.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s")
.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
// Do not use '%-0s' as a formatting conversion, since it will throw a
// a MissingFormatWidthException if it is used in String.format().
// http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Formatter.html#intFlags
if(displayECPolicy){
int maxEC = 0;
for (PathData item : items) {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
maxEC = maxLength(maxEC, contentSummary.getErasureCodingPolicy());
}
fmt.append((maxEC > 0) ? "%-" + maxEC + "s " : "%s");
}
fmt.append("%" + maxLen + "s ");
fmt.append("%s %s"); // mod time & path
lineFormat = fmt.toString();
}
private int maxLength(int n, Object value) {
return Math.max(n, (value != null) ? String.valueOf(value).length() : 0);
}
/**
* Get the comparator to be used for sorting files.
* @return comparator
*/
private Comparator<PathData> getOrderComparator() {
return this.orderComparator;
}
/**
* Initialise the comparator to be used for sorting files. If multiple options
* are selected then the order is chosen in the following precedence: -
* Modification time (or access time if requested) - File size - File name
*/
private void initialiseOrderComparator() {
if (isOrderTime()) {
// mtime is ordered latest date first in line with the unix ls -t command
this.orderComparator = new Comparator<PathData>() {
public int compare(PathData o1, PathData o2) {
Long o1Time = (isUseAtime() ? o1.stat.getAccessTime()
: o1.stat.getModificationTime());
Long o2Time = (isUseAtime() ? o2.stat.getAccessTime()
: o2.stat.getModificationTime());
return o2Time.compareTo(o1Time) * (isOrderReverse() ? -1 : 1);
}
};
} else if (isOrderSize()) {
// size is ordered largest first in line with the unix ls -S command
this.orderComparator = new Comparator<PathData>() {
public int compare(PathData o1, PathData o2) {
Long o1Length = o1.stat.getLen();
Long o2Length = o2.stat.getLen();
return o2Length.compareTo(o1Length) * (isOrderReverse() ? -1 : 1);
}
};
} else {
this.orderComparator = new Comparator<PathData>() {
public int compare(PathData o1, PathData o2) {
return o1.compareTo(o2) * (isOrderReverse() ? -1 : 1);
}
};
}
}
/**
* Get a recursive listing of all files in that match the file patterns.
* Same as "-ls -R"
*/
public static class Lsr extends Ls {
public static final String NAME = "lsr";
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
args.addFirst("-R");
super.processOptions(args);
}
@Override
public String getReplacementCommand() {
return "ls -R";
}
}
}
相关信息
相关文章
hadoop CommandWithDestination 源码
0
赞
热门推荐
-
2、 - 优质文章
-
3、 gate.io
-
6、 golang
-
7、 openharmony