hadoop LayoutVersion 源码

haddop LayoutVersion 代码

文件路径：/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.protocol;

import java.util.Comparator;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.hadoop.classification.InterfaceAudience;

/**
 * This class tracks changes in the layout version of HDFS.
 * 
 * Layout version is changed for following reasons:
 * <ol>
 * <li>The layout of how namenode or datanode stores information 
 * on disk changes.</li>
 * <li>A new operation code is added to the editlog.</li>
 * <li>Modification such as format of a record, content of a record 
 * in editlog or fsimage.</li>
 * </ol>
 * <br>
 * <b>How to update layout version:<br></b>
 * When a change requires new layout version, please add an entry into
 * {@link Feature} with a short enum name, new layout version and description
 * of the change. Please see {@link Feature} for further details.
 * <br>
 */
@InterfaceAudience.Private
public class LayoutVersion {
  /**
   * Version in which HDFS-2991 was fixed. This bug caused OP_ADD to
   * sometimes be skipped for append() calls. If we see such a case when
   * loading the edits, but the version is known to have that bug, we
   * workaround the issue. Otherwise we should consider it a corruption
   * and bail.
   */
  public static final int BUGFIX_HDFS_2991_VERSION = -40;

  /**
   * The interface to be implemented by NameNode and DataNode layout features 
   */
  public interface LayoutFeature {
    public FeatureInfo getInfo();
  }

  /**
   * Enums for features that change the layout version before rolling
   * upgrade is supported.
   * <br><br>
   * To add a new layout version:
   * <ul>
   * <li>Define a new enum constant with a short enum name, the new layout version 
   * and description of the added feature.</li>
   * <li>When adding a layout version with an ancestor that is not same as
   * its immediate predecessor, use the constructor where a specific ancestor
   * can be passed.
   * </li>
   * </ul>
   */
  public enum Feature implements LayoutFeature {
    NAMESPACE_QUOTA(-16, "Support for namespace quotas"),
    FILE_ACCESS_TIME(-17, "Support for access time on files"),
    DISKSPACE_QUOTA(-18, "Support for disk space quotas"),
    STICKY_BIT(-19, "Support for sticky bits"),
    APPEND_RBW_DIR(-20, "Datanode has \"rbw\" subdirectory for append"),
    ATOMIC_RENAME(-21, "Support for atomic rename"),
    CONCAT(-22, "Support for concat operation"),
    SYMLINKS(-23, "Support for symbolic links"),
    DELEGATION_TOKEN(-24, "Support for delegation tokens for security"),
    FSIMAGE_COMPRESSION(-25, "Support for fsimage compression"),
    FSIMAGE_CHECKSUM(-26, "Support checksum for fsimage"),
    REMOVE_REL13_DISK_LAYOUT_SUPPORT(-27, "Remove support for 0.13 disk layout"),
    EDITS_CHECKSUM(-28, "Support checksum for editlog"),
    UNUSED(-29, "Skipped version"),
    FSIMAGE_NAME_OPTIMIZATION(-30, "Store only last part of path in fsimage"),
    RESERVED_REL20_203(-31, -19, "Reserved for release 0.20.203", true,
        DELEGATION_TOKEN),
    RESERVED_REL20_204(-32, -31, "Reserved for release 0.20.204", true),
    RESERVED_REL22(-33, -27, "Reserved for release 0.22", true),
    RESERVED_REL23(-34, -30, "Reserved for release 0.23", true),
    FEDERATION(-35, "Support for namenode federation"),
    LEASE_REASSIGNMENT(-36, "Support for persisting lease holder reassignment"),
    STORED_TXIDS(-37, "Transaction IDs are stored in edits log and image files"),
    TXID_BASED_LAYOUT(-38, "File names in NN Storage are based on transaction IDs"), 
    EDITLOG_OP_OPTIMIZATION(-39,
        "Use LongWritable and ShortWritable directly instead of ArrayWritable of UTF8"),
    OPTIMIZE_PERSIST_BLOCKS(-40,
        "Serialize block lists with delta-encoded variable length ints, " +
        "add OP_UPDATE_BLOCKS"),
    RESERVED_REL1_2_0(-41, -32, "Reserved for release 1.2.0", true, CONCAT),
    ADD_INODE_ID(-42, -40, "Assign a unique inode id for each inode", false),
    SNAPSHOT(-43, "Support for snapshot feature"),
    RESERVED_REL1_3_0(-44, -41, "Reserved for release 1.3.0", true,
    		ADD_INODE_ID, SNAPSHOT, FSIMAGE_NAME_OPTIMIZATION),
    OPTIMIZE_SNAPSHOT_INODES(-45, -43,
        "Reduce snapshot inode memory footprint", false),
    SEQUENTIAL_BLOCK_ID(-46, "Allocate block IDs sequentially and store " +
        "block IDs in the edits log and image files"),
    EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to " 
        + "enable rebuilding retry cache in case of HA failover"),
    EDITLOG_ADD_BLOCK(-48, "Add new editlog that only records allocation of "
        + "the new block instead of the entire block list"),
    ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid."
        + " Use distinct StorageUuid per storage directory."),
    ADD_LAYOUT_FLAGS(-50, "Add support for layout flags."),
    CACHING(-51, "Support for cache pools and path-based caching"),
    // Hadoop 2.4.0
    PROTOBUF_FORMAT(-52, "Use protobuf to serialize FSImage"),
    EXTENDED_ACL(-53, "Extended ACL"),
    RESERVED_REL2_4_0(-54, -51, "Reserved for release 2.4.0", true,
        PROTOBUF_FORMAT, EXTENDED_ACL);

    private final FeatureInfo info;

    /**
     * Feature that is added at layout version {@code lv} - 1. 
     * @param lv new layout version with the addition of this feature
     * @param description description of the feature
     */
    Feature(final int lv, final String description) {
      this(lv, lv + 1, description, false);
    }

    /**
     * Feature that is added at layout version {@code ancestoryLV}.
     * @param lv new layout version with the addition of this feature
     * @param ancestorLV layout version from which the new lv is derived from.
     * @param description description of the feature
     * @param reserved true when this is a layout version reserved for previous
     *        version
     * @param features set of features that are to be enabled for this version
     */
    Feature(final int lv, final int ancestorLV, final String description,
        boolean reserved, Feature... features) {
      info = new FeatureInfo(lv, ancestorLV, description, reserved, features);
    }
    
    @Override
    public FeatureInfo getInfo() {
      return info;
    }
  }
  
  /** Feature information. */
  public static class FeatureInfo {
    private final int lv;
    private final int ancestorLV;
    private final Integer minCompatLV;
    private final String description;
    private final boolean reserved;
    private final LayoutFeature[] specialFeatures;

    public FeatureInfo(final int lv, final int ancestorLV, final String description,
        boolean reserved, LayoutFeature... specialFeatures) {
      this(lv, ancestorLV, null, description, reserved, specialFeatures);
    }

    public FeatureInfo(final int lv, final int ancestorLV, Integer minCompatLV,
        final String description, boolean reserved,
        LayoutFeature... specialFeatures) {
      this.lv = lv;
      this.ancestorLV = ancestorLV;
      this.minCompatLV = minCompatLV;
      this.description = description;
      this.reserved = reserved;
      this.specialFeatures = specialFeatures;
    }
    
    /** 
     * Accessor method for feature layout version 
     * @return int lv value
     */
    public int getLayoutVersion() {
      return lv;
    }

    /** 
     * Accessor method for feature ancestor layout version 
     * @return int ancestor LV value
     */
    public int getAncestorLayoutVersion() {
      return ancestorLV;
    }

    /**
     * Accessor method for feature minimum compatible layout version.  If the
     * feature does not define a minimum compatible layout version, then this
     * method returns the feature's own layout version.  This would indicate
     * that the feature cannot provide compatibility with any prior layout
     * version.
     *
     * @return int minimum compatible LV value
     */
    public int getMinimumCompatibleLayoutVersion() {
      return minCompatLV != null ? minCompatLV : lv;
    }

    /**
     * Accessor method for feature description 
     * @return String feature description 
     */
    public String getDescription() {
      return description;
    }
    
    public boolean isReservedForOldRelease() {
      return reserved;
    }
    
    public LayoutFeature[] getSpecialFeatures() {
      return specialFeatures;
    }
  }

  static class LayoutFeatureComparator implements Comparator<LayoutFeature> {
    @Override
    public int compare(LayoutFeature arg0, LayoutFeature arg1) {
      return arg0.getInfo().getLayoutVersion()
          - arg1.getInfo().getLayoutVersion();
    }
  }
 
  public static void updateMap(Map<Integer, SortedSet<LayoutFeature>> map,
      LayoutFeature[] features) {
    // Go through all the enum constants and build a map of
    // LayoutVersion <-> Set of all supported features in that LayoutVersion
    SortedSet<LayoutFeature> existingFeatures = new TreeSet<LayoutFeature>(
        new LayoutFeatureComparator());
    for (SortedSet<LayoutFeature> s : map.values()) {
      existingFeatures.addAll(s);
    }
    LayoutFeature prevF = existingFeatures.isEmpty() ? null :
        existingFeatures.first();
    for (LayoutFeature f : features) {
      final FeatureInfo info = f.getInfo();
      int minCompatLV = info.getMinimumCompatibleLayoutVersion();
      if (prevF != null &&
          minCompatLV > prevF.getInfo().getMinimumCompatibleLayoutVersion()) {
        throw new AssertionError(String.format(
            "Features must be listed in order of minimum compatible layout " +
            "version.  Check features %s and %s.", prevF, f));
      }
      prevF = f;
      SortedSet<LayoutFeature> ancestorSet = map.get(info.getAncestorLayoutVersion());
      if (ancestorSet == null) {
        // Empty set
        ancestorSet = new TreeSet<LayoutFeature>(new LayoutFeatureComparator());
        map.put(info.getAncestorLayoutVersion(), ancestorSet);
      }
      SortedSet<LayoutFeature> featureSet = new TreeSet<LayoutFeature>(ancestorSet);
      if (info.getSpecialFeatures() != null) {
        for (LayoutFeature specialFeature : info.getSpecialFeatures()) {
          featureSet.add(specialFeature);
        }
      }
      featureSet.add(f);
      map.put(info.getLayoutVersion(), featureSet);
    }
  }
  
  /**
   * Gets formatted string that describes {@link LayoutVersion} information.
   */
  public String getString(Map<Integer, SortedSet<LayoutFeature>> map,
      LayoutFeature[] values) {
    final StringBuilder buf = new StringBuilder();
    buf.append("Feature List:\n");
    for (LayoutFeature f : values) {
      final FeatureInfo info = f.getInfo();
      buf.append(f).append(" introduced in layout version ")
          .append(info.getLayoutVersion()).append(" (")
          .append(info.getDescription()).append(")\n");
    }

    buf.append("\n\nLayoutVersion and supported features:\n");
    for (LayoutFeature f : values) {
      final FeatureInfo info = f.getInfo();
      buf.append(info.getLayoutVersion()).append(": ")
          .append(map.get(info.getLayoutVersion())).append("\n");
    }
    return buf.toString();
  }
  
  /**
   * Returns true if a given feature is supported in the given layout version
   * @param map layout feature map
   * @param f Feature
   * @param lv LayoutVersion
   * @return true if {@code f} is supported in layout version {@code lv}
   */
  public static boolean supports(Map<Integer, SortedSet<LayoutFeature>> map,
      final LayoutFeature f, final int lv) {
    final SortedSet<LayoutFeature> set =  map.get(lv);
    return set != null && set.contains(f);
  }
  
  /**
   * Get the current layout version
   */
  public static int getCurrentLayoutVersion(LayoutFeature[] features) {
    return getLastNonReservedFeature(features).getInfo().getLayoutVersion();
  }

  /**
   * Gets the minimum compatible layout version.
   *
   * @param features all features to check
   * @return minimum compatible layout version
   */
  public static int getMinimumCompatibleLayoutVersion(
      LayoutFeature[] features) {
    return getLastNonReservedFeature(features).getInfo()
        .getMinimumCompatibleLayoutVersion();
  }

  static LayoutFeature getLastNonReservedFeature(LayoutFeature[] features) {
    for (int i = features.length -1; i >= 0; i--) {
      final FeatureInfo info = features[i].getInfo();
      if (!info.isReservedForOldRelease()) {
        return features[i];
      }
    }
    throw new AssertionError("All layout versions are reserved.");
  }
}

相关信息

hadoop 源码目录

相关文章

hadoop BlockListAsLongs 源码

hadoop CacheDirective 源码

hadoop FSLimitException 源码

hadoop LayoutFlags 源码

hadoop RecoveryInProgressException 源码

hadoop RollingUpgradeException 源码

hadoop SnapshotException 源码

hadoop SnapshotInfo 源码

hadoop UnregisteredNodeException 源码