hadoop DefaultResourceCalculator 源码

haddop DefaultResourceCalculator 代码

文件路径：/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java

/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.util.resource;

import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;

import java.util.Set;

@Private
@Unstable
public class DefaultResourceCalculator extends ResourceCalculator {
  private static final Logger LOG =
      LoggerFactory.getLogger(DefaultResourceCalculator.class);

  private static final Set<String> INSUFFICIENT_RESOURCE_NAME =
      ImmutableSet.of(ResourceInformation.MEMORY_URI);

  @Override
  public int compare(Resource unused, Resource lhs, Resource rhs,
      boolean singleType) {
    // Only consider memory
    return Long.compare(lhs.getMemorySize(), rhs.getMemorySize());
  }

  @Override
  public long computeAvailableContainers(Resource available, Resource required) {
    // Only consider memory
    return available.getMemorySize() / required.getMemorySize();
  }

  @Override
  public float divide(Resource unused, 
      Resource numerator, Resource denominator) {
    return ratio(numerator, denominator);
  }
  
  public boolean isInvalidDivisor(Resource r) {
    if (r.getMemorySize() == 0.0f) {
      return true;
    }
    return false;
  }

  @Override
  public float ratio(Resource a, Resource b) {
    return divideSafelyAsFloat(a.getMemorySize(), b.getMemorySize());
  }

  @Override
  public Resource divideAndCeil(Resource numerator, int denominator) {
    return Resources.createResource(
        divideAndCeil(numerator.getMemorySize(), denominator));
  }

  @Override
  public Resource divideAndCeil(Resource numerator, float denominator) {
    return Resources.createResource(
        divideAndCeil(numerator.getMemorySize(), denominator));
  }

  @Override
  public Resource normalize(Resource r, Resource minimumResource,
      Resource maximumResource, Resource stepFactor) {
    if (stepFactor.getMemorySize() == 0) {
      LOG.error("Memory cannot be allocated in increments of zero. Assuming " +
          minimumResource.getMemorySize() + "MB increment size. "
          + "Please ensure the scheduler configuration is correct.");
      stepFactor = minimumResource;
    }

    long normalizedMemory = Math.min(
        roundUp(
            Math.max(r.getMemorySize(), minimumResource.getMemorySize()),
            stepFactor.getMemorySize()),
            maximumResource.getMemorySize());
    return Resources.createResource(normalizedMemory);
  }

  @Override
  public Resource roundUp(Resource r, Resource stepFactor) {
    return Resources.createResource(
        roundUp(r.getMemorySize(), stepFactor.getMemorySize())
        );
  }

  @Override
  public Resource roundDown(Resource r, Resource stepFactor) {
    return Resources.createResource(
        roundDown(r.getMemorySize(), stepFactor.getMemorySize()));
  }

  @Override
  public Resource multiplyAndNormalizeUp(Resource r, double by,
      Resource stepFactor) {
    return Resources.createResource(
        roundUp((long) (r.getMemorySize() * by + 0.5),
            stepFactor.getMemorySize()));
  }

  @Override
  public Resource multiplyAndNormalizeUp(Resource r, double[] by,
      Resource stepFactor) {
    return Resources.createResource(
        roundUp((long) (r.getMemorySize() * by[0] + 0.5),
            stepFactor.getMemorySize()));
  }

  @Override
  public Resource multiplyAndNormalizeDown(Resource r, double by,
      Resource stepFactor) {
    return Resources.createResource(
        roundDown(
            (long)(r.getMemorySize() * by),
            stepFactor.getMemorySize()
            )
        );
  }

  @Override
  public boolean fitsIn(Resource smaller, Resource bigger) {
    return smaller.getMemorySize() <= bigger.getMemorySize();
  }

  @Override
  public Resource normalizeDown(Resource r, Resource stepFactor) {
    return Resources.createResource(
        roundDown((r.getMemorySize()), stepFactor.getMemorySize()));
  }

  @Override
  public boolean isAnyMajorResourceZeroOrNegative(Resource resource) {
    return resource.getMemorySize() <= 0;
  }

  @Override
  public boolean isAnyMajorResourceAboveZero(Resource resource) {
    return resource.getMemorySize() > 0;
  }

  public Set<String> getInsufficientResourceNames(Resource required,
      Resource available) {
    if (required.getMemorySize() > available.getMemorySize()) {
      return INSUFFICIENT_RESOURCE_NAME;
    } else {
      return ImmutableSet.of();
    }
  }

  @Override
  public boolean isAllInvalidDivisor(Resource r) {
    return isInvalidDivisor(r);
  }
}

相关信息

hadoop 源码目录

相关文章

hadoop DominantResourceCalculator 源码

hadoop ResourceCalculator 源码

hadoop Resources 源码