background

Time: 13:58:26, February 26, 2021

Brief steps:

  1. Determine what the command is?
  2. What is the location of the command?
  3. What is the main class for the command to execute?
  4. Follow the execution process to view

Example: HDFS dfsadmin-report storage index and HDFS dfs-du h/results are not consistent, need to check the statistical logic difference between the two files

  1. Determine the location of the command, which is HDFS

[ops@m-onedata bin]$ which is hdfs

/usr/bin/hdfs

  1. Look at the script, cat /usr/bin/hdfs

exec /usr/… /hadoop-hdfs/bin/hdfs.distro “$@”

Discover the location of the script that was actually executed, and proceed to the execution script

elif [ “$COMMAND” = “dfs” ] ; then

CLASS=org.apache.hadoop.fs.FsShell

HADOOP_OPTS=”$HADOOP_OPTS $HADOOP_CLIENT_OPTS”

elif [ “$COMMAND” = “dfsadmin” ] ; then

CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin

HADOOP_OPTS=”$HADOOP_OPTS $HADOOP_CLIENT_OPTS”

elif [ “$COMMAND” = “haadmin” ] ; then

CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin

CLASSPATH=${CLASSPATH}:${TOOL_PATH}

HADOOP_OPTS=”$HADOOP_OPTS $HADOOP_CLIENT_OPTS”

elif [ “$COMMAND” = “fsck” ] ; then

CLASS=org.apache.hadoop.hdfs.tools.DFSck

HADOOP_OPTS=”$HADOOP_OPTS $HADOOP_CLIENT_OPTS”

elif [ “$COMMAND” = “balancer” ] ; then

CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer

HADOOP_OPTS=”$HADOOP_OPTS $HADOOP_BALANCER_OPTS”

exec “$JAVA” -Dproc_$COMMAND $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS “$@”

  1. Find the main classes FsShell and DFSAdmin that are actually executed
  2. View the process being executed

HDFS dfsadmin-report storage index and HDFS dfs-du-h/results are not consistent, we need to check the differences between the statistical logic of the two files

[hadoop@123-onedata ~]# HDFS dfsadmin-report Configured Capacity: 157902726414336 (143.61TB) Configured Capacity: 143526582003116 (130.54TB) DFS Remaining: 18651259047864 (16.96TB) DFS Used: 124875322955252 (113.57TB) DFS Used%: Corrupt replicas: 0 Missing replicas 209619 Missing blocks (with replication factor 1): 0 [hadoop@123-onedata ~]$HDFS dfs-du-h / 33.8g/MR-history 74.9m /spark2-history 18.6t /user

Screening process

Find the main classes FsShell and DFSAdmin that are actually executed

hdfs dfs -du

FsShell

/** * main() has some simple utility methods * @param argv the command and its arguments * @throws Exception upon error */ public static void main(String argv[]) throws Exception { FsShell shell = newShellInstance(); // Create an FSShell object Configuration conf = new Configuration(); conf.setQuietMode(false); shell.setConf(conf); int res; try { res = ToolRunner.run(shell, argv); } finally {shell.close(); } System.exit(res); } / / -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- - line -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- run / * * * * / @ Override public int run(String[] argv) { // initialize FsShell init(); Tracer tracer = new Tracer.Builder("FsShell"). conf(TraceUtils.wrapHadoopConf(SHELL_HTRACE_PREFIX, getConf())). build(); int exitCode = -1; if (argv.length < 1) { printUsage(System.err); } else { String cmd = argv[0]; Command instance = null; try { instance = commandFactory.getInstance(cmd); If (instance == null) {throw new UnknownCommandException(); } TraceScope scope = tracer.newScope(instance.getCommandName()); if (scope.getSpan() ! = null) { String args = StringUtils.join(" ", argv); if (args.length() > 2048) { args = args.substring(0, 2048); } scope.getSpan().addKVAnnotation("args", args); } try { exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length)); } finally {scope.close(); } } catch (IllegalArgumentException e) { if (e.getMessage() == null) { displayError(cmd, "Null exception message"); e.printStackTrace(System.err); } else { displayError(cmd, e.getLocalizedMessage()); } printUsage(System.err); if (instance ! = null) { printInstanceUsage(System.err, instance); } } catch (Exception e) { // instance.run catches IOE, so something is REALLY wrong if here LOG.debug("Error", e); displayError(cmd, "Fatal internal error"); e.printStackTrace(System.err); } } tracer.close(); return exitCode; }

CommandFactory

/** * Get an instance of the requested command * @param cmdName name of the command to lookup * @param conf the hadoop configuration * @return the {@link Command} or null if the command is unknown */ public Command getInstance(String cmdName, Configuration conf) { if (conf == null) throw new NullPointerException("configuration is null"); Command instance = objectMap.get(cmdName); If (instance == null) {Class<? extends Command> cmdClass = classMap.get(cmdName); If (cmdClass! = cmdClass! = cmdClass! = cmdClass! = null) { instance = ReflectionUtils.newInstance(cmdClass, conf); instance.setName(cmdName); instance.setCommandFactory(this); } } return instance; }

Since it is the command object that executes the corresponding command, then find the corresponding object and check the inheritance tree of the command as follows:

DFSAdminCommand in DFSAdmin (org.apache.hadoop.hdfs.tools) SetSpaceQuotaCommand in DFSAdmin (org.apache.hadoop.hdfs.tools) ClearSpaceQuotaCommand in DFSAdmin (org.apache.hadoop.hdfs.tools) SetQuotaCommand in DFSAdmin (org.apache.hadoop.hdfs.tools) ClearQuotaCommand in DFSAdmin (org.apache.hadoop.hdfs.tools) FsCommand (org.apache.hadoop.fs.shell) Stat (org.apache.hadoop.fs.shell) SetfaclCommand in AclCommands (org.apache.hadoop.fs.shell) Mkdir (org.apache.hadoop.fs.shell) Display (org.apache.hadoop.fs.shell) FsShellPermissions (org.apache.hadoop.fs) Help in FsShell (org.apache.hadoop.fs) Truncate (org.apache.hadoop.fs.shell) Ls (org.apache.hadoop.fs.shell) Usage in FsShell (org.apache.hadoop.fs) Rmdir in Delete (org.apache.hadoop.fs.shell) DeleteSnapshot in SnapshotCommands (org.apache.hadoop.fs.shell) Find (org.apache.hadoop.fs.shell.find) InterruptCommand in TestFsShellReturnCode (org.apache.hadoop.fs) SetReplication (org.apache.hadoop.fs.shell) Count (org.apache.hadoop.fs.shell) SnapshotCommands (org.apache.hadoop.fs.shell) AclCommands (org.apache.hadoop.fs.shell) TouchCommands (org.apache.hadoop.fs.shell) RenameSnapshot in SnapshotCommands (org.apache.hadoop.fs.shell) FsUsage (org.apache.hadoop.fs.shell) XAttrCommands (org.apache.hadoop.fs.shell) Merge in CopyCommands (org.apache.hadoop.fs.shell) MoveToLocal in MoveCommands (org.apache.hadoop.fs.shell) GetfattrCommand in XAttrCommands (org.apache.hadoop.fs.shell) CommandWithDestination (org.apache.hadoop.fs.shell) Rm in Delete (org.apache.hadoop.fs.shell) Expunge in Delete (org.apache.hadoop.fs.shell) CreateSnapshot in SnapshotCommands (org.apache.hadoop.fs.shell) GetfaclCommand in AclCommands (org.apache.hadoop.fs.shell) SetfattrCommand in XAttrCommands  (org.apache.hadoop.fs.shell) Head (org.apache.hadoop.fs.shell) Tail (org.apache.hadoop.fs.shell) Test (org.apache.hadoop.fs.shell)

We can easily find the Command corresponding to du by using the class search function of Idea

Du

Override protected void procesPath (PathData Item) throws IOException {// Accumulate ContentSummary of all files by subdirectories of the loop directory contentSummary = item.fs.getContentSummary(item.path); long length = contentSummary.getLength(); long spaceConsumed = contentSummary.getSpaceConsumed(); if (excludeSnapshots) { length -= contentSummary.getSnapshotLength(); spaceConsumed -= contentSummary.getSnapshotSpaceConsumed(); } getUsagesTable().addRow(formatSize(length), formatSize(spaceConsumed), item); }

FileSystem

/** Return the {@link ContentSummary} of a given {@link Path}. * @param f path to use * @throws FileNotFoundException if  the path does not resolve * @throws IOException IO failure */ public ContentSummary getContentSummary(Path f) throws IOException { FileStatus status = getFileStatus(f); if (status.isFile()) { // f is a file long length = status.getLen(); return new ContentSummary.Builder().length(length). fileCount(1).directoryCount(0).spaceConsumed(length).build(); } // f is a directory long[] summary = {0, 0, 1}; for(FileStatus s : listStatus(f)) { long length = s.getLen(); ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : new ContentSummary.Builder().length(length). fileCount(1).directoryCount(0).spaceConsumed(length).build(); summary[0] += c.getLength(); summary[1] += c.getFileCount(); summary[2] += c.getDirectoryCount(); } // In fact, we can see the total number of directories available, the total file size, Return new ContentSummary.Builder().Length (Summary [1]).FileCount (Summary [1]).DirectoryCount (Summary [2]). spaceConsumed(summary[0]).build(); }

hdfs dfsadmin -report

DFSAdmin

/** * main() has some simple utility methods. * @param argv Command line parameters. * @exception Exception if the filesystem does not exist. */ public static void main(String[] argv) throws Exception { int res = ToolRunner.run(new DFSAdmin(), argv); // Execute the System.exit(res) method of the command object; } /** * @param argv The parameters passed to this program. * @exception Exception if the filesystem does not exist. * @return 0 on success, non zero on error. */ @Override public int run(String[] argv) { if (argv.length < 1) { printUsage(""); return -1; } / /... Try {if ("-report".equals(CMD)) {report(argv, I); } else if ("-safemode".equals(CMD)) {}} //... } /** * Gives a report on how the FileSystem is doing.fs; /** * Gives a report on how the FileSystem is doing.fs * @Exception ioException if the filesystem does not exist. */ public void report(String[] argv, argv, argv) int i) throws IOException { DistributedFileSystem dfs = getDFS(); FsStatus ds = dfs.getStatus(); long capacity = ds.getCapacity(); long used = ds.getUsed(); Long Remaining = ds.getRemaining(); fsRemaining = ds.getRemaining(); fsRemaining = fsRemaining (); long bytesInFuture = dfs.getBytesWithFutureGenerationStamps(); long presentCapacity = used + remaining; boolean mode = dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET); if (mode) { System.out.println("Safe mode is ON"); if (bytesInFuture > 0) { System.out.println("\nWARNING: "); System.out.println("Name node has detected blocks with generation " + "stamps in future."); System.out.println("Forcing exit from safemode will cause " + bytesInFuture + " byte(s) to be deleted."); System.out.println("If you are sure that the NameNode was started with" + " the correct metadata files then you may proceed with " + "'-safemode forceExit'\n"); } } System.out.println("Configured Capacity: " + capacity + " (" + StringUtils.byteDesc(capacity) + ")"); System.out.println("Present Capacity: " + presentCapacity + " (" + StringUtils.byteDesc(presentCapacity) + ")"); System.out.println("DFS Remaining: " + remaining + " (" + StringUtils.byteDesc(remaining) + ")"); System.out.println("DFS Used: " + used + " (" + StringUtils.byteDesc(used) + ")"); double dfsUsedPercent = 0; if (presentCapacity ! = 0) { dfsUsedPercent = used/(double)presentCapacity; } System.out.println("DFS Used%: " + StringUtils.formatPercent(dfsUsedPercent, 2)); /* These counts are not always upto date. They are updated after * iteration of an internal list. Should be updated in a  few seconds to * minutes. Use "-metaSave" to list of all such blocks and accurate * counts. */ ReplicatedBlockStats replicatedBlockStats = dfs.getClient().getNamenode().getReplicatedBlockStats(); System.out.println("Replicated Blocks:"); System.out.println("\tUnder replicated blocks: " + replicatedBlockStats.getLowRedundancyBlocks()); System.out.println("\tBlocks with corrupt replicas: " + replicatedBlockStats.getCorruptBlocks()); System.out.println("\tMissing blocks: " + replicatedBlockStats.getMissingReplicaBlocks()); System.out.println("\tMissing blocks (with replication factor 1): " + replicatedBlockStats.getMissingReplicationOneBlocks()); if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) { System.out.println("\tLow redundancy blocks with highest priority " + "to recover: " + replicatedBlockStats.getHighestPriorityLowRedundancyBlocks()); } // Omit some useless code here}

DistributedFileSystem

Override public fsStatus getStatus(Path p) throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_STATUS); return dfs.getDiskStatus(); }

DFSClient

  /**
   * @see ClientProtocol#getStats()
   */
  public FsStatus getDiskStatus() throws IOException {
    return new FsStatus(getStateByIndex(0),
        getStateByIndex(1), getStateByIndex(2));
  }

NameNodeRpcServer

  @Override // ClientProtocol
  public long[] getStats() throws IOException {
    checkNNStartup();
    namesystem.checkOperation(OperationCategory.READ);
    return namesystem.getStats();
  }

FSNamesystem

/** @see ClientProtocol#getStats() */ long[] getStats() { final long[] stats = datanodeStatistics.getStats(); [CLIENTPROTOCO.GET_STATS_LOW_RENDEX_IDX] = GETLOWRENDEX_RENDEX_IDX] = GETLOWRENDEX_RENDEX_IDX; stats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] = getCorruptReplicaBlocks(); stats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] = getMissingBlocksCount(); stats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] = getMissingReplOneBlocksCount(); stats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX] = blockManager.getBytesInFuture(); stats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX] = blockManager.getPendingDeletionBlocksCount(); return stats; }