hadoop SnapshotFSImageFormat 源码

  • 2022-10-20
  • 浏览 (44)

haddop SnapshotFSImageFormat 代码


 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *     http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * See the License for the specific language governing permissions and
 * limitations under the License.
package org.apache.hadoop.hdfs.server.namenode.snapshot;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.apache.hadoop.hdfs.util.ReadOnlyList;

import org.apache.hadoop.util.Preconditions;

 * A helper class defining static methods for reading/writing snapshot related
 * information from/to FSImage.
public class SnapshotFSImageFormat {
   * Save snapshots and snapshot quota for a snapshottable directory.
   * @param current The directory that the snapshots belongs to.
   * @param out The {@link DataOutput} to write.
   * @throws IOException
  public static void saveSnapshots(INodeDirectory current, DataOutput out)
      throws IOException {
    DirectorySnapshottableFeature sf = current.getDirectorySnapshottableFeature();
    Preconditions.checkArgument(sf != null);
    // list of snapshots in snapshotsByNames
    ReadOnlyList<Snapshot> snapshots = sf.getSnapshotList();
    for (Snapshot s : snapshots) {
      // write the snapshot id
    // snapshot quota

   * Save SnapshotDiff list for an INodeDirectoryWithSnapshot.
   * @param sNode The directory that the SnapshotDiff list belongs to.
   * @param out The {@link DataOutput} to write.
  private static <N extends INode, A extends INodeAttributes, D extends AbstractINodeDiff<N, A, D>>
      void saveINodeDiffs(final AbstractINodeDiffList<N, A, D> diffs,
      final DataOutput out, ReferenceMap referenceMap) throws IOException {
    // Record the diffs in reversed order, so that we can find the correct
    // reference for INodes in the created list when loading the FSImage
    if (diffs == null) {
      out.writeInt(-1); // no diffs
    } else {
      final DiffList<D> list = diffs.asList();
      final int size = list.size();
      for (int i = size - 1; i >= 0; i--) {
        list.get(i).write(out, referenceMap);

  public static void saveDirectoryDiffList(final INodeDirectory dir,
      final DataOutput out, final ReferenceMap referenceMap
      ) throws IOException {
    saveINodeDiffs(dir.getDiffs(), out, referenceMap);

  public static void saveFileDiffList(final INodeFile file,
      final DataOutput out) throws IOException {
    saveINodeDiffs(file.getDiffs(), out, null);

  public static FileDiffList loadFileDiffList(DataInput in,
      FSImageFormat.Loader loader) throws IOException {
    final int size = in.readInt();
    if (size == -1) {
      return null;
    } else {
      final FileDiffList diffs = new FileDiffList();
      FileDiff posterior = null;
      for(int i = 0; i < size; i++) {
        final FileDiff d = loadFileDiff(posterior, in, loader);
        posterior = d;
      return diffs;

  private static FileDiff loadFileDiff(FileDiff posterior, DataInput in,
      FSImageFormat.Loader loader) throws IOException {
    // 1. Read the id of the Snapshot root to identify the Snapshot
    final Snapshot snapshot = loader.getSnapshot(in);

    // 2. Load file size
    final long fileSize = in.readLong();
    // 3. Load snapshotINode 
    final INodeFileAttributes snapshotINode = in.readBoolean()?
        loader.loadINodeFileAttributes(in): null;
    return new FileDiff(snapshot.getId(), snapshotINode, posterior, fileSize);

   * Load a node stored in the created list from fsimage.
   * @param createdNodeName The name of the created node.
   * @param parent The directory that the created list belongs to.
   * @return The created node.
  public static INode loadCreated(byte[] createdNodeName,
      INodeDirectory parent) throws IOException {
    // the INode in the created list should be a reference to another INode
    // in posterior SnapshotDiffs or one of the current children
    for (DirectoryDiff postDiff : parent.getDiffs()) {
      final INode d = postDiff.getChildrenDiff().getDeleted(createdNodeName);
      if (d != null) {
        return d;
      } // else go to the next SnapshotDiff
    // use the current child
    INode currentChild = parent.getChild(createdNodeName,
    if (currentChild == null) {
      throw new IOException("Cannot find an INode associated with the INode "
          + DFSUtil.bytes2String(createdNodeName)
          + " in created list while loading FSImage.");
    return currentChild;
   * Load the created list from fsimage.
   * @param parent The directory that the created list belongs to.
   * @param in The {@link DataInput} to read.
   * @return The created list.
  private static List<INode> loadCreatedList(INodeDirectory parent,
      DataInput in) throws IOException {
    // read the size of the created list
    int createdSize = in.readInt();
    List<INode> createdList = new ArrayList<INode>(createdSize);
    for (int i = 0; i < createdSize; i++) {
      byte[] createdNodeName = FSImageSerialization.readLocalName(in);
      INode created = loadCreated(createdNodeName, parent);
    return createdList;
   * Load the deleted list from the fsimage.
   * @param parent The directory that the deleted list belongs to.
   * @param createdList The created list associated with the deleted list in 
   *                    the same Diff.
   * @param in The {@link DataInput} to read.
   * @param loader The {@link Loader} instance.
   * @return The deleted list.
  private static List<INode> loadDeletedList(INodeDirectory parent,
      List<INode> createdList, DataInput in, FSImageFormat.Loader loader)
      throws IOException {
    int deletedSize = in.readInt();
    List<INode> deletedList = new ArrayList<INode>(deletedSize);
    for (int i = 0; i < deletedSize; i++) {
      final INode deleted = loader.loadINodeWithLocalName(true, in, true);
      // set parent: the parent field of an INode in the deleted list is not 
      // useful, but set the parent here to be consistent with the original 
      // fsdir tree.
      if (deleted.isFile()) {
    return deletedList;
   * Load snapshots and snapshotQuota for a Snapshottable directory.
   * @param snapshottableParent
   *          The snapshottable directory for loading.
   * @param numSnapshots
   *          The number of snapshots that the directory has.
   * @param loader
   *          The loader
  public static void loadSnapshotList(INodeDirectory snapshottableParent,
      int numSnapshots, DataInput in, FSImageFormat.Loader loader)
      throws IOException {
    DirectorySnapshottableFeature sf = snapshottableParent
    Preconditions.checkArgument(sf != null);
    for (int i = 0; i < numSnapshots; i++) {
      // read snapshots
      final Snapshot s = loader.getSnapshot(in);
    int snapshotQuota = in.readInt();

   * Load the {@link SnapshotDiff} list for the INodeDirectoryWithSnapshot
   * directory.
   * @param dir
   *          The snapshottable directory for loading.
   * @param in
   *          The {@link DataInput} instance to read.
   * @param loader
   *          The loader
  public static void loadDirectoryDiffList(INodeDirectory dir,
      DataInput in, FSImageFormat.Loader loader) throws IOException {
    final int size = in.readInt();
    if (dir.isWithSnapshot()) {
      DirectoryDiffList diffs = dir.getDiffs();
      for (int i = 0; i < size; i++) {
        diffs.addFirst(loadDirectoryDiff(dir, in, loader));

   * Load the snapshotINode field of {@link AbstractINodeDiff}.
   * @param snapshot The Snapshot associated with the {@link AbstractINodeDiff}.
   * @param in The {@link DataInput} to read.
   * @param loader The {@link Loader} instance that this loading procedure is
   *               using.
   * @return The snapshotINode.
  private static INodeDirectoryAttributes loadSnapshotINodeInDirectoryDiff(
      Snapshot snapshot, DataInput in, FSImageFormat.Loader loader)
      throws IOException {
    // read the boolean indicating whether snapshotINode == Snapshot.Root
    boolean useRoot = in.readBoolean();      
    if (useRoot) {
      return snapshot.getRoot();
    } else {
      // another boolean is used to indicate whether snapshotINode is non-null
      return in.readBoolean()? loader.loadINodeDirectoryAttributes(in): null;
   * Load {@link DirectoryDiff} from fsimage.
   * @param parent The directory that the SnapshotDiff belongs to.
   * @param in The {@link DataInput} instance to read.
   * @param loader The {@link Loader} instance that this loading procedure is 
   *               using.
   * @return A {@link DirectoryDiff}.
  private static DirectoryDiff loadDirectoryDiff(INodeDirectory parent,
      DataInput in, FSImageFormat.Loader loader) throws IOException {
    // 1. Read the full path of the Snapshot root to identify the Snapshot
    final Snapshot snapshot = loader.getSnapshot(in);

    // 2. Load DirectoryDiff#childrenSize
    int childrenSize = in.readInt();
    // 3. Load DirectoryDiff#snapshotINode 
    INodeDirectoryAttributes snapshotINode = loadSnapshotINodeInDirectoryDiff(
        snapshot, in, loader);
    // 4. Load the created list in SnapshotDiff#Diff
    List<INode> createdList = loadCreatedList(parent, in);
    // 5. Load the deleted list in SnapshotDiff#Diff
    List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);
    // 6. Compose the SnapshotDiff
    DiffList<DirectoryDiff> diffs = parent.getDiffs().asList();
    DirectoryDiff sdiff = new DirectoryDiff(snapshot.getId(), snapshotINode,
        diffs.isEmpty() ? null : diffs.get(0), childrenSize, createdList,
        deletedList, snapshotINode == snapshot.getRoot());
    return sdiff;

  /** A reference map for fsimage serialization. */
  public static class ReferenceMap {
     * Used to indicate whether the reference node itself has been saved
    private final Map<Long, INodeReference.WithCount> referenceMap
        = new HashMap<Long, INodeReference.WithCount>();
     * Used to record whether the subtree of the reference node has been saved 
    private final Map<Long, Long> dirMap = new HashMap<Long, Long>();

    public void writeINodeReferenceWithCount(
        INodeReference.WithCount withCount, DataOutput out,
        boolean writeUnderConstruction) throws IOException {
      final INode referred = withCount.getReferredINode();
      final long id = withCount.getId();
      final boolean firstReferred = !referenceMap.containsKey(id);

      if (firstReferred) {
        FSImageSerialization.saveINode2Image(referred, out,
            writeUnderConstruction, this);
        referenceMap.put(id, withCount);
      } else {
    public boolean toProcessSubtree(long id) {
      if (dirMap.containsKey(id)) {
        return false;
      } else {
        dirMap.put(id, id);
        return true;
    public INodeReference.WithCount loadINodeReferenceWithCount(
        boolean isSnapshotINode, DataInput in, FSImageFormat.Loader loader
        ) throws IOException {
      final boolean firstReferred = in.readBoolean();

      final INodeReference.WithCount withCount;
      if (firstReferred) {
        final INode referred = loader.loadINodeWithLocalName(isSnapshotINode,
            in, true);
        withCount = new INodeReference.WithCount(null, referred);
        referenceMap.put(withCount.getId(), withCount);
      } else {
        final long id = in.readLong();
        withCount = referenceMap.get(id);
      return withCount;


hadoop 源码目录


hadoop AbstractINodeDiff 源码

hadoop AbstractINodeDiffList 源码

hadoop DiffList 源码

hadoop DiffListByArrayList 源码

hadoop DiffListBySkipList 源码

hadoop DirectoryDiffListFactory 源码

hadoop DirectorySnapshottableFeature 源码

hadoop DirectoryWithSnapshotFeature 源码

hadoop FSImageFormatPBSnapshot 源码

hadoop FileDiff 源码

0  赞