hadoop ConfigurationProperties 源码

haddop ConfigurationProperties 代码

文件路径：/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfigurationProperties.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

/**
 * A trie storage to preprocess and store configuration properties for optimised
 * retrieval. A node is created for every key part delimited by ".".
 * A property entry is stored in a node that matches its next to last key
 * part (which reduces the nodes created).
 * For example:
 * yarn.scheduler.capacity.root.max-applications 100
 * yarn.scheduler.capacity.root.state RUNNING
 * 4 nodes are created: yarn - scheduler - capacity - root
 * root node will have the two properties set in its values.
 */
public class ConfigurationProperties {
  private static final Logger LOG =
      LoggerFactory.getLogger(ConfigurationProperties.class);

  private final Map<String, PrefixNode> nodes;
  private static final String DELIMITER = "\\.";

  /**
   * A constructor defined in order to conform to the type used by
   * {@code Configuration}. It must only be called by String keys and values.
   * @param props properties to store
   */
  public ConfigurationProperties(Map<String, String> props) {
    this.nodes = new HashMap<>();
    storePropertiesInPrefixNodes(props);
  }

  /**
   * Filters all properties by a prefix. The property keys are trimmed by the
   * given prefix.
   * @param prefix prefix to filter property keys
   * @return properties matching given prefix
   */
  public Map<String, String> getPropertiesWithPrefix(String prefix) {
    return getPropertiesWithPrefix(prefix, false);
  }

  /**
   * Filters all properties by a prefix.
   * @param prefix prefix to filter property keys
   * @param fullyQualifiedKey whether collected property keys are to be trimmed
   *                          by the prefix, or must be kept as it is
   * @return properties matching given prefix
   */
  public Map<String, String> getPropertiesWithPrefix(
      String prefix, boolean fullyQualifiedKey) {
    List<String> propertyPrefixParts = splitPropertyByDelimiter(prefix);
    Map<String, String> properties = new HashMap<>();
    String trimPrefix;
    if (fullyQualifiedKey) {
      trimPrefix = "";
    } else {
      // To support the behaviour where the
      // CapacitySchedulerConfiguration.getQueuePrefix(String queue) method
      // returned with the queue prefix with a dot appended to it the last dot
      // should be removed
      trimPrefix = prefix.endsWith(CapacitySchedulerConfiguration.DOT) ?
          prefix.substring(0, prefix.length() - 1) : prefix;
    }

    collectPropertiesRecursively(nodes, properties,
        propertyPrefixParts.iterator(), trimPrefix);

    return properties;
  }

  /**
   * Collects properties stored in all nodes that match the given prefix.
   * @param childNodes children to consider when collecting properties
   * @param properties aggregated property storage
   * @param prefixParts prefix parts split by delimiter
   * @param trimPrefix a string that needs to be trimmed from the collected
   *                   property, empty if the key must be kept as it is
   */
  private void collectPropertiesRecursively(
      Map<String, PrefixNode> childNodes, Map<String, String> properties,
      Iterator<String> prefixParts, String trimPrefix) {
    if (prefixParts.hasNext()) {
      String prefix = prefixParts.next();
      PrefixNode candidate = childNodes.get(prefix);

      if (candidate != null) {
        if (!prefixParts.hasNext()) {
          copyProperties(properties, trimPrefix, candidate.getValues());
        }
        collectPropertiesRecursively(candidate.getChildren(), properties,
            prefixParts, trimPrefix);
      }
    } else {
      for (Map.Entry<String, PrefixNode> child : childNodes.entrySet()) {
        copyProperties(properties, trimPrefix, child.getValue().getValues());
        collectPropertiesRecursively(child.getValue().getChildren(),
            properties, prefixParts, trimPrefix);
      }
    }
  }


  /**
   * Copy properties stored in a node to an aggregated property storage.
   * @param copyTo property storage that collects processed properties stored
   *               in nodes
   * @param trimPrefix a string that needs to be trimmed from the collected
   *                   property, empty if the key must be kept as it is
   * @param copyFrom properties stored in a node
   */
  private void copyProperties(
      Map<String, String> copyTo, String trimPrefix,
      Map<String, String> copyFrom) {
    for (Map.Entry<String, String> configEntry : copyFrom.entrySet()) {
      String key = configEntry.getKey();
      String prefixToTrim = trimPrefix;

      if (!trimPrefix.isEmpty()) {
        if (!key.equals(trimPrefix)) {
          prefixToTrim += CapacitySchedulerConfiguration.DOT;
        }
        key = configEntry.getKey().substring(prefixToTrim.length());
      }

      copyTo.put(key, configEntry.getValue());
    }
  }

  /**
   * Stores the given properties in the correct node.
   * @param props properties that need to be stored
   */
  private void storePropertiesInPrefixNodes(Map<String, String> props) {
    for (Map.Entry<String, String> prop : props.entrySet()) {
      List<String> propertyKeyParts = splitPropertyByDelimiter(prop.getKey());
      if (!propertyKeyParts.isEmpty()) {
        PrefixNode node = findOrCreatePrefixNode(nodes,
            propertyKeyParts.iterator());
        node.getValues().put(prop.getKey(), prop.getValue());
      } else {
        LOG.warn("Empty configuration property, skipping...");
      }
    }
  }

  /**
   * Finds the node that matches the whole key or create it, if it does not
   * exist.
   * @param children child nodes on current level
   * @param propertyKeyParts a property key split by delimiter
   * @return the last node
   */
  private PrefixNode findOrCreatePrefixNode(
      Map<String, PrefixNode> children, Iterator<String> propertyKeyParts) {
    String prefix = propertyKeyParts.next();
    PrefixNode candidate = children.get(prefix);
    if (candidate == null) {
      candidate = new PrefixNode();
      children.put(prefix, candidate);
    }

    if (!propertyKeyParts.hasNext()) {
      return candidate;
    }

    return findOrCreatePrefixNode(candidate.getChildren(),
        propertyKeyParts);
  }

  private List<String> splitPropertyByDelimiter(String property) {
    return Arrays.asList(property.split(DELIMITER));
  }


  /**
   * A node that represents a prefix part. For example:
   * yarn.scheduler consists of a "yarn" and a "scheduler" node.
   * children: contains the child nodes, like "yarn" has a "scheduler" child
   * values: contains the actual property key-value pairs with this prefix.
   */
  private static class PrefixNode {
    private final Map<String, String> values;
    private final Map<String, PrefixNode> children;

    PrefixNode() {
      this.values = new HashMap<>();
      this.children = new HashMap<>();
    }

    public Map<String, String> getValues() {
      return values;
    }

    public Map<String, PrefixNode> getChildren() {
      return children;
    }
  }
}

相关信息

hadoop 源码目录

相关文章

hadoop AbstractAutoCreatedLeafQueue 源码

hadoop AbstractCSQueue 源码

hadoop AbstractLeafQueue 源码

hadoop AbstractManagedParentQueue 源码

hadoop AppPriorityACLConfigurationParser 源码

hadoop AppPriorityACLGroup 源码

hadoop AutoCreatedLeafQueue 源码

hadoop AutoCreatedLeafQueueConfig 源码

hadoop AutoCreatedQueueDeletionPolicy 源码

hadoop AutoCreatedQueueManagementPolicy 源码