hadoop LegacyMappingRuleToJson 源码

haddop LegacyMappingRuleToJson 代码

文件路径：/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/converter/LegacyMappingRuleToJson.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.converter;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;

import java.util.ArrayList;
import java.util.Collection;

public class LegacyMappingRuleToJson {
  //Legacy rule parse helper constants
  public static final String RULE_PART_DELIMITER = ":";
  public static final String PREFIX_USER_MAPPING = "u";
  public static final String PREFIX_GROUP_MAPPING = "g";

  //Legacy rule matcher variables
  public static final String MATCHER_APPLICATION = "%application";
  public static final String MATCHER_USER = "%user";

  //Legacy rule mapping variables, which can be used in target queues
  public static final String MAPPING_PRIMARY_GROUP = "%primary_group";
  public static final String MAPPING_SECONDARY_GROUP = "%secondary_group";
  public static final String MAPPING_USER = MATCHER_USER;

  //JSON Format match all token (actually only used for users)
  public static final String JSON_MATCH_ALL = "*";

  //Frequently used JSON node names for rule definitions
  public static final String JSON_NODE_POLICY = "policy";
  public static final String JSON_NODE_PARENT_QUEUE = "parentQueue";
  public static final String JSON_NODE_CUSTOM_PLACEMENT = "customPlacement";
  public static final String JSON_NODE_MATCHES = "matches";

  /**
   * Our internal object mapper, used to create JSON nodes.
   */
  private ObjectMapper objectMapper = new ObjectMapper();

  /**
   * Collection to store the legacy group mapping rule strings.
   */
  private Collection<String> userGroupMappingRules = new ArrayList<>();
  /**
   * Collection to store the legacy application name mapping rule strings.
   */
  private Collection<String> applicationNameMappingRules = new ArrayList<>();

  /**
   * This setter method is used to set the raw string format of the legacy
   * user group mapping rules. This method expect a string formatted just like
   * in the configuration file of the Capacity Scheduler.
   * eg. u:bob:root.groups.%primary_group,u:%user:root.default
   *
   * @param rules The string containing ALL the UserGroup mapping rules in
   *              legacy format
   * @return This object for daisy chain support
   */
  public LegacyMappingRuleToJson setUserGroupMappingRules(String rules) {
    setUserGroupMappingRules(StringUtils.getTrimmedStringCollection(rules));
    return this;
  }

  /**
   * This setter method is used to set the the user group mapping rules as a
   * string collection, where each entry is one rule.
   *
   * @param rules One rule per entry
   * @return This object for daisy chain support
   */
  public LegacyMappingRuleToJson setUserGroupMappingRules(
      Collection<String> rules) {
    if (rules != null) {
      userGroupMappingRules = rules;
    } else {
      userGroupMappingRules = new ArrayList<>();
    }
    return this;
  }

  /**
   * This setter method is used to set the raw string format of the legacy
   * application name mapping rules. This method expect a string formatted
   * just like in the configuration file of the Capacity Scheduler.
   * eg. mapreduce:root.apps.%application,%application:root.default
   *
   * @param rules The string containing ALL the application name mapping rules
   *              in legacy format
   * @return This object for daisy chain support
   */
  public LegacyMappingRuleToJson setAppNameMappingRules(String rules) {
    setAppNameMappingRules(StringUtils.getTrimmedStringCollection(rules));
    return this;
  }

  /**
   * This setter method is used to set the the application name mapping rules as
   * a string collection, where each entry is one rule.
   *
   * @param rules One rule per entry
   * @return This object for daisy chain support
   */
  public LegacyMappingRuleToJson setAppNameMappingRules(
      Collection<String> rules) {
    if (rules != null) {
      applicationNameMappingRules = rules;
    } else {
      applicationNameMappingRules = new ArrayList<>();
    }

    return this;
  }

  /**
   * This method will do the conversion based on the already set mapping rules.
   * First the rules to be converted must be set via setAppNameMappingRules and
   * setUserGroupMappingRules methods.
   * @return JSON Format of the provided mapping rules, null if no rules are set
   */
  public String convert() {
    //creating the basic JSON config structure
    ObjectNode rootNode = objectMapper.createObjectNode();
    ArrayNode rulesNode = objectMapper.createArrayNode();
    rootNode.set("rules", rulesNode);

    //Processing and adding all the user group mapping rules
    for (String rule : userGroupMappingRules) {
      rulesNode.add(convertUserGroupMappingRule(rule));
    }

    //Processing and adding all the application name mapping rules
    for (String rule : applicationNameMappingRules) {
      rulesNode.add(convertAppNameMappingRule(rule));
    }

    //If there are no converted rules we return null
    if (rulesNode.size() == 0) {
      return null;
    }

    try {
      return objectMapper
          .writerWithDefaultPrettyPrinter()
          .writeValueAsString(rootNode);
    } catch (JsonProcessingException e) {
      e.printStackTrace();
    }

    return null;
  }

  /**
   * This intermediate helper method is used to process User Group mapping rules
   * and invoke the proper mapping rule creation method.
   * @param rule The legacy format of the single rule to be converted.
   * @return The ObjectNode which can be added to the rules part of the config.
   */
  ObjectNode convertUserGroupMappingRule(String rule) {
    String[] mapping = splitRule(rule, 3);
    String ruleType = mapping[0];
    String ruleMatch = mapping[1];
    String ruleTarget = mapping[2];

    if (ruleType.equals(PREFIX_USER_MAPPING)) {
      return createUserMappingRule(ruleMatch, ruleTarget);
    }

    if (ruleType.equals(PREFIX_GROUP_MAPPING)) {
      return createGroupMappingRule(ruleMatch, ruleTarget);
    }

    throw new IllegalArgumentException(
        "User group mapping rule must start with prefix '" +
            PREFIX_USER_MAPPING + "' or '" + PREFIX_GROUP_MAPPING + "'");
  }

  /**
   * This intermediate helper method is used to process Application name mapping
   * rules and invoke the proper mapping rule creation method.
   * @param rule The legacy format of the single rule to be converted.
   * @return The ObjectNode which can be added to the rules part of the config.
   */
  ObjectNode convertAppNameMappingRule(String rule) {
    String[] mapping = splitRule(rule, 2);
    String ruleMatch = mapping[0];
    String ruleTarget = mapping[1];

    return createApplicationNameMappingRule(ruleMatch, ruleTarget);
  }
  /**
   * Helper method which splits the rules into parts, and checks if it has
   * exactly the required amount of parts, and none of them is empty!
   * @param rule The mapping rule to be split
   * @param expectedParts The number of expected parts
   * @return The split String[] of the parts
   * @throws IllegalArgumentException if the number of parts don't match or any
   *  of them is empty.
   */
  private String[] splitRule(String rule, int expectedParts) {
    //Splitting
    String[] mapping = StringUtils
        .getTrimmedStringCollection(rule, RULE_PART_DELIMITER)
        .toArray(new String[] {});

    //Checking for part count
    if (mapping.length != expectedParts) {
      throw new IllegalArgumentException("Invalid rule '" + rule +
          "' expected parts: " + expectedParts +
          " actual parts: " + mapping.length);
    }

    //Checking for empty parts
    for (int i = 0; i < mapping.length; i++) {
      if (mapping[i].length() == 0) {
        throw new IllegalArgumentException("Invalid rule '" + rule +
            "' with empty part, mapping rules must not contain empty parts!");
      }
    }

    return mapping;
  }

  /**
   * This helper method is to create a default rule node for the converter,
   * setting fields which are common in all rules.
   * @param type The type of the rule can be user/group/application
   * @return The object node with the preset fields
   */
  private ObjectNode createDefaultRuleNode(String type) {
    return objectMapper
        .createObjectNode()
        .put("type", type)
        //All legacy rule fallback to place to default
        .put("fallbackResult", "placeDefault")
        //All legacy rules allow creation
        .put("create", true);
  }

  /**
   * This method will create the JSON node for a single User Mapping Rule.
   * @param match The match part of the rule it can be either an actual user
   *              name or '%user' to match all users
   * @param target The queue to place to user into, some queue path variables
   *               are supported (%user, %primary_group, %secondary_group).
   * @return The ObjectNode which represents the rule
   */
  private ObjectNode createUserMappingRule(String match, String target) {
    ObjectNode ruleNode = createDefaultRuleNode("user");
    QueuePath targetPath = new QueuePath(target);

    //We have a special token in the JSON format to match all user, replacing
    //matcher
    if (match.equals(MATCHER_USER)) {
      match = JSON_MATCH_ALL;
    }
    ruleNode.put(JSON_NODE_MATCHES, match);

    switch (targetPath.getLeafName()) {
    case MAPPING_USER:
      ruleNode.put(JSON_NODE_POLICY, "user");
      if (targetPath.hasParent()) {
        //Parsing parent path, to be able to determine the short name of parent
        QueuePath targetParentPath =
            new QueuePath(targetPath.getParent());
        String parentShortName = targetParentPath.getLeafName();

        if (parentShortName.equals(MAPPING_PRIMARY_GROUP)) {
          //%primary_group.%user mapping
          ruleNode.put(JSON_NODE_POLICY, "primaryGroupUser");

          //Yep, this is confusing. The policy primaryGroupUser actually
          // appends the %primary_group.%user to the parent path, so we need to
          // remove it from the parent path to avoid duplication.
          targetPath = new QueuePath(targetParentPath.getParent(),
              targetPath.getLeafName());
        } else if (parentShortName.equals(MAPPING_SECONDARY_GROUP)) {
          //%secondary_group.%user mapping
          ruleNode.put(JSON_NODE_POLICY, "secondaryGroupUser");

          //Yep, this is confusing. The policy secondaryGroupUser actually
          // appends the %secondary_group.%user to the parent path, so we need
          // to remove it from the parent path to avoid duplication.
          targetPath = new QueuePath(targetParentPath.getParent(),
              targetPath.getLeafName());
        }

        //[parent].%user mapping
      }
      break;
    case MAPPING_PRIMARY_GROUP:
      //[parent].%primary_group mapping
      ruleNode.put(JSON_NODE_POLICY, "primaryGroup");
      break;
    case MAPPING_SECONDARY_GROUP:
      //[parent].%secondary_group mapping
      ruleNode.put(JSON_NODE_POLICY, "secondaryGroup");
      break;
    default:
      //static path mapping
      ruleNode.put(JSON_NODE_POLICY, "custom");
      ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
      break;
    }

    //if the target queue has a parent part, and the rule can have a parent
    //we add it to the node
    if (targetPath.hasParent()) {
      ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
    }

    return ruleNode;
  }

  /**
   * This method will create the JSON node for a single Group Mapping Rule.
   * @param match The name of the group to match for
   * @param target The queue to place to user into, some queue path variables
   *               are supported (%user).
   * @return The ObjectNode which represents the rule
   */
  private ObjectNode createGroupMappingRule(String match, String target) {
    ObjectNode ruleNode = createDefaultRuleNode("group");
    QueuePath targetPath = new QueuePath(target);

    //we simply used the source match part all valid legacy matchers are valid
    //matchers for the JSON format as well
    ruleNode.put(JSON_NODE_MATCHES, match);

    if (targetPath.getLeafName().matches(MATCHER_USER)) {
      //g:group:[parent].%user mapping
      ruleNode.put(JSON_NODE_POLICY, "user");

      //if the target queue has a parent part we add it to the node
      if (targetPath.hasParent()) {
        ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
      }
    } else {
      //static path mapping
      ruleNode.put(JSON_NODE_POLICY, "custom");
      ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
    }

    return ruleNode;
  }


  /**
   * This method will create the JSON node for a single Application Name
   * Mapping Rule.
   * @param match The name of the application to match for or %application to
   *              match all applications
   * @param target The queue to place to user into, some queue path variables
   *               are supported (%application).
   * @return The ObjectNode which represents the rule
   */
  private ObjectNode createApplicationNameMappingRule(
      String match, String target) {
    ObjectNode ruleNode = createDefaultRuleNode("application");
    QueuePath targetPath = new QueuePath(target);

    //we simply used the source match part all valid legacy matchers are valid
    //matchers for the JSON format as well
    ruleNode.put(JSON_NODE_MATCHES, match);

    if (targetPath.getLeafName().matches(MATCHER_APPLICATION)) {
      //[parent].%application mapping
      ruleNode.put(JSON_NODE_POLICY, "applicationName");

      //if the target queue has a parent part we add it to the node
      if (targetPath.hasParent()) {
        ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
      }
    } else {
      //static path mapping
      ruleNode.put(JSON_NODE_POLICY, "custom");
      ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
    }

    return ruleNode;
  }
}

相关信息

hadoop 源码目录

相关文章

hadoop BufferPool 源码

hadoop ByteBufferInputStream 源码

hadoop ByteBufferOutputStream 源码

hadoop ByteBufferWrapper 源码

hadoop Constants 源码

hadoop CosN 源码

hadoop CosNConfigKeys 源码

hadoop CosNCopyFileContext 源码

hadoop CosNCopyFileTask 源码

hadoop CosNFileReadTask 源码