hadoop ConfTest 源码

haddop ConfTest 代码

文件路径：/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.util;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Stack;
import java.util.Map.Entry;

import javax.xml.namespace.QName;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.Attribute;
import javax.xml.stream.events.Characters;
import javax.xml.stream.events.StartElement;
import javax.xml.stream.events.XMLEvent;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.classification.InterfaceAudience;

/**
 * This class validates configuration XML files in ${HADOOP_CONF_DIR} or
 * specified ones.
 */
@InterfaceAudience.Private
public final class ConfTest {

  private static final String USAGE =
      "Usage: hadoop conftest [-conffile <path>|-h|--help]\n"
      + "  Options:\n"
      + "  \n"
      + "  -conffile <path>\n"
      + "    If not specified, the files in ${HADOOP_CONF_DIR}\n"
      + "    whose name end with .xml will be verified.\n"
      + "    If specified, that path will be verified.\n"
      + "    You can specify either a file or directory, and\n"
      + "    if a directory specified, the files in that directory\n"
      + "    whose name end with .xml will be verified.\n"
      + "    You can specify this option multiple times.\n"
      + "  -h, --help       Print this help";

  private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";

  protected ConfTest() {
    super();
  }

  private static List<NodeInfo> parseConf(InputStream in)
      throws XMLStreamException {
    QName configuration = new QName("configuration");
    QName property = new QName("property");

    List<NodeInfo> nodes = new ArrayList<NodeInfo>();
    Stack<NodeInfo> parsed = new Stack<>();

    XMLInputFactory factory = XMLInputFactory.newInstance();
    XMLEventReader reader = factory.createXMLEventReader(in);

    while (reader.hasNext()) {
      XMLEvent event = reader.nextEvent();
      if (event.isStartElement()) {
        StartElement currentElement = event.asStartElement();
        NodeInfo currentNode = new NodeInfo(currentElement);
        if (parsed.isEmpty()) {
          if (!currentElement.getName().equals(configuration)) {
            return null;
          }
        } else {
          NodeInfo parentNode = parsed.peek();
          QName parentName = parentNode.getStartElement().getName();
          if (parentName.equals(configuration)
              && currentNode.getStartElement().getName().equals(property)) {
            @SuppressWarnings("unchecked")
            Iterator<Attribute> it = currentElement.getAttributes();
            while (it.hasNext()) {
              currentNode.addAttribute(it.next());
            }
          } else if (parentName.equals(property)) {
            parentNode.addElement(currentElement);
          }
        }
        parsed.push(currentNode);
      } else if (event.isEndElement()) {
        NodeInfo node = parsed.pop();
        if (parsed.size() == 1) {
          nodes.add(node);
        }
      } else if (event.isCharacters()) {
        if (2 < parsed.size()) {
          NodeInfo parentNode = parsed.pop();
          StartElement parentElement = parentNode.getStartElement();
          NodeInfo grandparentNode = parsed.peek();
          if (grandparentNode.getElement(parentElement) == null) {
            grandparentNode.setElement(parentElement, event.asCharacters());
          }
          parsed.push(parentNode);
        }
      }
    }

    return nodes;
  }

  public static List<String> checkConf(InputStream in) {
    List<NodeInfo> nodes = null;
    List<String> errors = new ArrayList<String>();

    try {
      nodes = parseConf(in);
      if (nodes == null) {
        errors.add("bad conf file: top-level element not <configuration>");
      }
    } catch (XMLStreamException e) {
      errors.add("bad conf file: " + e.getMessage());
    }

    if (!errors.isEmpty()) {
      return errors;
    }

    Map<String, List<Integer>> duplicatedProperties =
        new HashMap<String, List<Integer>>();

    for (NodeInfo node : nodes) {
      StartElement element = node.getStartElement();
      int line = element.getLocation().getLineNumber();

      if (!element.getName().equals(new QName("property"))) {
        errors.add(String.format("Line %d: element not <property>", line));
        continue;
      }

      List<XMLEvent> events = node.getXMLEventsForQName(new QName("name"));
      if (events == null) {
        errors.add(String.format("Line %d: <property> has no <name>", line));
      } else {
        String v = null;
        for (XMLEvent event : events) {
          if (event.isAttribute()) {
            v = ((Attribute) event).getValue();
          } else {
            Characters c = node.getElement(event.asStartElement());
            if (c != null) {
              v = c.getData();
            }
          }
          if (v == null || v.isEmpty()) {
            errors.add(String.format("Line %d: <property> has an empty <name>",
                line));
          }
        }
        if (v != null && !v.isEmpty()) {
          List<Integer> lines = duplicatedProperties.get(v);
          if (lines == null) {
            lines = new ArrayList<Integer>();
            duplicatedProperties.put(v, lines);
          }
          lines.add(node.getStartElement().getLocation().getLineNumber());
        }
      }

      events = node.getXMLEventsForQName(new QName("value"));
      if (events == null) {
        errors.add(String.format("Line %d: <property> has no <value>", line));
      }

      for (QName qName : node.getDuplicatedQNames()) {
        if (!qName.equals(new QName("source"))) {
          errors.add(String.format("Line %d: <property> has duplicated <%s>s",
              line, qName));
        }
      }
    }

    for (Entry<String, List<Integer>> e : duplicatedProperties.entrySet()) {
      List<Integer> lines = e.getValue();
      if (1 < lines.size()) {
        errors.add(String.format("Line %s: duplicated <property>s for %s",
            StringUtils.join(", ", lines), e.getKey()));
      }
    }

    return errors;
  }

  private static File[] listFiles(File dir) {
    return dir.listFiles(new FileFilter() {
      @Override
      public boolean accept(File file) {
        return file.isFile() && file.getName().endsWith(".xml");
      }
    });
  }

  @SuppressWarnings("static-access")
  public static void main(String[] args) throws IOException {
    GenericOptionsParser genericParser = new GenericOptionsParser(args);
    String[] remainingArgs = genericParser.getRemainingArgs();

    Option conf = OptionBuilder.hasArg().create("conffile");
    Option help = OptionBuilder.withLongOpt("help").create('h');
    Options opts = new Options().addOption(conf).addOption(help);
    CommandLineParser specificParser = new GnuParser();
    CommandLine cmd = null;
    try {
      cmd = specificParser.parse(opts, remainingArgs);
    } catch (MissingArgumentException e) {
      terminate(1, "No argument specified for -conffile option");
    } catch (ParseException e) {
      terminate(1, USAGE);
    }
    if (cmd == null) {
      terminate(1, "Failed to parse options");
    }

    if (cmd.hasOption('h')) {
      terminate(0, USAGE);
    }

    List<File> files = new ArrayList<File>();
    if (cmd.hasOption("conffile")) {
      String[] values = cmd.getOptionValues("conffile");
      for (String value : values) {
        File confFile = new File(value);
        if (confFile.isFile()) {
          files.add(confFile);
        } else if (confFile.isDirectory()) {
          files.addAll(Arrays.asList(listFiles(confFile)));
        } else {
          terminate(1, confFile.getAbsolutePath()
              + " is neither a file nor directory");
        }
      }
    } else {
      String confDirName = System.getenv(HADOOP_CONF_DIR);
      if (confDirName == null) {
        terminate(1, HADOOP_CONF_DIR + " is not defined");
      }
      File confDir = new File(confDirName);
      if (!confDir.isDirectory()) {
        terminate(1, HADOOP_CONF_DIR + " is not a directory");
      }
      files = Arrays.asList(listFiles(confDir));
    }
    if (files.isEmpty()) {
      terminate(1, "No input file to validate");
    }

    boolean ok = true;
    for (File file : files) {
      String path = file.getAbsolutePath();
      List<String> errors = checkConf(Files.newInputStream(file.toPath()));
      if (errors.isEmpty()) {
        System.out.println(path + ": valid");
      } else {
        ok = false;
        System.err.println(path + ":");
        for (String error : errors) {
          System.err.println("\t" + error);
        }
      }
    }
    if (ok) {
      System.out.println("OK");
    } else {
      terminate(1, "Invalid file exists");
    }
  }

  private static void terminate(int status, String msg) {
    System.err.println(msg);
    System.exit(status);
  }
}

class NodeInfo {

  private StartElement startElement;
  private List<Attribute> attributes = new ArrayList<Attribute>();
  private Map<StartElement, Characters> elements =
      new HashMap<>();
  private Map<QName, List<XMLEvent>> qNameXMLEventsMap =
      new HashMap<>();

  public NodeInfo(StartElement startElement) {
    this.startElement = startElement;
  }

  private void addQNameXMLEvent(QName qName, XMLEvent event) {
    List<XMLEvent> events = qNameXMLEventsMap.get(qName);
    if (events == null) {
      events = new ArrayList<XMLEvent>();
      qNameXMLEventsMap.put(qName, events);
    }
    events.add(event);
  }

  public StartElement getStartElement() {
    return startElement;
  }

  public void addAttribute(Attribute attribute) {
    attributes.add(attribute);
    addQNameXMLEvent(attribute.getName(), attribute);
  }

  public Characters getElement(StartElement element) {
    return elements.get(element);
  }

  public void addElement(StartElement element) {
    setElement(element, null);
    addQNameXMLEvent(element.getName(), element);
  }

  public void setElement(StartElement element, Characters text) {
    elements.put(element, text);
  }

  public List<QName> getDuplicatedQNames() {
    List<QName> duplicates = new ArrayList<QName>();
    for (Map.Entry<QName, List<XMLEvent>> e : qNameXMLEventsMap.entrySet()) {
      if (1 < e.getValue().size()) {
        duplicates.add(e.getKey());
      }
    }
    return duplicates;
  }

  public List<XMLEvent> getXMLEventsForQName(QName qName) {
    return qNameXMLEventsMap.get(qName);
  }
}

相关信息

hadoop 源码目录

相关文章

hadoop ApplicationClassLoader 源码

hadoop AsyncDiskService 源码

hadoop AutoCloseableLock 源码

hadoop BasicDiskValidator 源码

hadoop BlockingThreadPoolExecutorService 源码

hadoop CacheableIPList 源码

hadoop ChunkedArrayList 源码

hadoop ClassUtil 源码

hadoop Classpath 源码

hadoop CleanerUtil 源码