hadoop JavaSandboxLinuxContainerRuntime 源码
haddop JavaSandboxLinuxContainerRuntime 代码
文件路径：/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.
containermanager.linux.runtime;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FilePermission;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.AllPermission;
import java.util.Formatter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.hadoop.fs.Path.SEPARATOR;
import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCALIZED_RESOURCES;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
/**
* <p>This class extends the {@link DefaultLinuxContainerRuntime} specifically
* for containers which run Java commands. It generates a new java security
* policy file per container and modifies the java command to enable the
* Java Security Manager with the generated policy.</p>
*
* The behavior of the {@link JavaSandboxLinuxContainerRuntime} can be modified
* using the following settings:
*
* <ul>
* <li>
* {@value
* org.apache.hadoop.yarn.conf.YarnConfiguration#YARN_CONTAINER_SANDBOX} :
* This yarn-site.xml setting has three options:
* <ul>
* <li>disabled - Default behavior. {@link LinuxContainerRuntime}
* is disabled</li>
* <li>permissive - JVM containers will run with Java Security Manager
* enabled. Non-JVM containers will run normally</li>
* <li>enforcing - JVM containers will run with Java Security Manager
* enabled. Non-JVM containers will be prevented from executing and an
* {@link ContainerExecutionException} will be thrown.</li>
* </ul>
* </li>
* <li>
* {@value
* org.apache.hadoop.yarn.conf.YarnConfiguration#YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS}
* :
* Determines the file permissions for the application directories. The
* permissions come in the form of comma separated values
* (e.g. read,write,execute,delete). Defaults to {@code read} for read-only.
* </li>
* <li>
* {@value
* org.apache.hadoop.yarn.conf.YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY}
* :
* Accepts canonical path to a java policy file on the local filesystem.
* This file will be loaded as the base policy, any additional container
* grants will be appended to this base file. If not specified, the default
* java.policy file provided with hadoop resources will be used.
* </li>
* <li>
* {@value
* org.apache.hadoop.yarn.conf.YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP}
* :
* Optional setting to specify a YARN queue which will be exempt from the
* sand-boxing process.
* </li>
* <li>
* {@value
* org.apache.hadoop.yarn.conf.YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName
* :
* Optional setting to map groups to java policy files. The value is a path
* to the java policy file for $groupName. A user which is a member of
* multiple groups with different policies will receive the superset of all
* the permissions across their groups.
* </li>
* </ul>
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class JavaSandboxLinuxContainerRuntime
extends DefaultLinuxContainerRuntime {
private static final Logger LOG =
LoggerFactory.getLogger(DefaultLinuxContainerRuntime.class);
private Configuration configuration;
private SandboxMode sandboxMode;
public static final String POLICY_FILE_DIR = "nm-sandbox-policies";
private static Path policyFileDir;
private static final FileAttribute<Set<PosixFilePermission>> POLICY_ATTR =
PosixFilePermissions.asFileAttribute(
PosixFilePermissions.fromString("rwxr-xr-x"));
private Map<String, Path> containerPolicies = new HashMap<>();
/**
* Create an instance using the given {@link PrivilegedOperationExecutor}
* instance for performing operations.
*
* @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor}
* instance
*/
public JavaSandboxLinuxContainerRuntime(
PrivilegedOperationExecutor privilegedOperationExecutor) {
super(privilegedOperationExecutor);
}
@Override
public void initialize(Configuration conf, Context nmContext)
throws ContainerExecutionException {
this.configuration = conf;
this.sandboxMode =
SandboxMode.get(
this.configuration.get(YARN_CONTAINER_SANDBOX,
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
super.initialize(conf, nmContext);
}
/**
* Initialize the Java Security Policy directory. Either creates the
* directory if it doesn't exist, or clears the contents of the directory if
* already created.
* @throws ContainerExecutionException If unable to resolve policy directory
*/
private void initializePolicyDir() throws ContainerExecutionException {
String hadoopTempDir = configuration.get("hadoop.tmp.dir");
if (hadoopTempDir == null) {
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
}
policyFileDir = Paths.get(hadoopTempDir, POLICY_FILE_DIR);
//Delete any existing policy files if the directory has already been created
if(Files.exists(policyFileDir)){
try (DirectoryStream<Path> stream =
Files.newDirectoryStream(policyFileDir)){
for(Path policyFile : stream){
Files.delete(policyFile);
}
}catch(IOException e){
throw new ContainerExecutionException("Unable to initialize policy "
+ "directory: " + e);
}
} else {
try {
policyFileDir = Files.createDirectories(
Paths.get(hadoopTempDir, POLICY_FILE_DIR), POLICY_ATTR);
} catch (IOException e) {
throw new ContainerExecutionException("Unable to create policy file " +
"directory: " + e);
}
}
}
/**
* Prior to environment from being written locally need to generate
* policy file which limits container access to a small set of directories.
* Additionally the container run command needs to be modified to include
* flags to enable the java security manager with the generated policy.
* <br>
* The Java Sandbox will be circumvented if the user is a member of the
* group specified in:
* {@value
* org.apache.hadoop.yarn.conf.YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP}
* and if they do not include the JVM flag
* <code>-Djava.security.manager</code>.
*
* @param ctx The {@link ContainerRuntimeContext} containing container
* setup properties.
* @throws ContainerExecutionException Exception thrown if temporary policy
* file directory can't be created, or if any exceptions occur during policy
* file parsing and generation.
*/
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
@SuppressWarnings("unchecked")
List<String> localDirs =
ctx.getExecutionAttribute(CONTAINER_LOCAL_DIRS);
@SuppressWarnings("unchecked")
Map<org.apache.hadoop.fs.Path, List<String>> resources =
ctx.getExecutionAttribute(LOCALIZED_RESOURCES);
@SuppressWarnings("unchecked")
List<String> commands =
ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
Map<String, String> env =
ctx.getContainer().getLaunchContext().getEnvironment();
String username =
ctx.getExecutionAttribute(USER);
if(!isSandboxContainerWhitelisted(username, commands)) {
String tmpDirBase = configuration.get("hadoop.tmp.dir");
if (tmpDirBase == null) {
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
}
try {
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
initializePolicyDir();
List<String> groupPolicyFiles =
getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER));
Path policyFilePath = Files.createFile(
Paths.get(policyFileDir.toString(),
containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
POLICY_ATTR);
try(OutputStream policyOutputStream =
Files.newOutputStream(policyFilePath)) {
containerPolicies.put(containerID, policyFilePath);
NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
localDirs, groupPolicyFiles, resources, configuration);
NMContainerPolicyUtils.appendSecurityFlags(
commands, env, policyFilePath, sandboxMode);
}
} catch (IOException e) {
throw new ContainerExecutionException(e);
}
}
}
@Override
public void launchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
try {
super.launchContainer(ctx);
} finally {
deletePolicyFiles(ctx);
}
}
@Override
public void relaunchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
try {
super.relaunchContainer(ctx);
} finally {
deletePolicyFiles(ctx);
}
}
/**
* Determine if JVMSandboxLinuxContainerRuntime should be used. This is
* decided based on the value of
* {@value
* org.apache.hadoop.yarn.conf.YarnConfiguration#YARN_CONTAINER_SANDBOX}
* @param env the environment variable settings for the operation
* @return true if Sandbox is requested, false otherwise
*/
@Override
public boolean isRuntimeRequested(Map<String, String> env) {
return sandboxMode != SandboxMode.disabled;
}
private static List<String> getGroupPolicyFiles(Configuration conf,
String user) throws ContainerExecutionException {
Groups groups = Groups.getUserToGroupsMappingService(conf);
Set<String> userGroups;
try {
userGroups = groups.getGroupsSet(user);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
return userGroups.stream()
.map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX
+ group))
.filter(groupPolicy -> groupPolicy != null)
.collect(Collectors.toList());
}
/**
* Determine if the container should be whitelisted (i.e. exempt from the
* Java Security Manager).
* @param username The name of the user running the container
* @param commands The list of run commands for the container
* @return boolean value denoting whether the container should be whitelisted.
* @throws ContainerExecutionException If container user can not be resolved
*/
private boolean isSandboxContainerWhitelisted(String username,
List<String> commands) throws ContainerExecutionException {
String whitelistGroup = configuration.get(
YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
Groups groups = Groups.getUserToGroupsMappingService(configuration);
Set<String> userGroups;
boolean isWhitelisted = false;
try {
userGroups = groups.getGroupsSet(username);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
if(whitelistGroup != null && userGroups.contains(whitelistGroup)) {
// If any command has security flag, whitelisting is disabled
for(String cmd : commands) {
if(cmd.contains(NMContainerPolicyUtils.SECURITY_FLAG)){
isWhitelisted = false;
break;
} else {
isWhitelisted = true;
}
}
}
return isWhitelisted;
}
/**
* Deletes policy files for container specified by parameter. Additionally
* this method will age off any stale policy files generated by
* {@link JavaSandboxLinuxContainerRuntime}
* @param ctx Container context for files to be deleted
* @throws ContainerExecutionException if unable to access or delete policy
* files or generated policy file directory
*/
private void deletePolicyFiles(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
try {
Files.delete(containerPolicies.remove(
ctx.getExecutionAttribute(CONTAINER_ID_STR)));
} catch (IOException e) {
throw new ContainerExecutionException("Unable to delete policy file: "
+ e);
}
}
/**
* Enumeration of the modes the JavaSandboxLinuxContainerRuntime can use.
* See {@link JavaSandboxLinuxContainerRuntime} for details on the
* behavior of each setting.
*/
public enum SandboxMode {
enforcing("enforcing"),
permissive("permissive"),
disabled("disabled");
private final String mode;
SandboxMode(String mode){
this.mode = mode;
}
public static SandboxMode get(String mode) {
if(enforcing.mode.equals(mode)) {
return enforcing;
} else if(permissive.mode.equals(mode)) {
return permissive;
} else {
return disabled;
}
}
public String toString(){
return mode;
}
}
/**
* Static utility class defining String constants and static methods for the
* use of the {@link JavaSandboxLinuxContainerRuntime}.
*/
static final class NMContainerPolicyUtils{
static final String POLICY_FILE = "java.policy";
static final String SECURITY_DEBUG = " -Djava.security.debug=all";
static final String SECURITY_FLAG = "-Djava.security.manager";
static final String POLICY_APPEND_FLAG = "-Djava.security.policy=";
static final String POLICY_FLAG = POLICY_APPEND_FLAG + "=";
static final String JAVA_CMD = "/bin/java ";
static final String JVM_SECURITY_CMD =
JAVA_CMD + SECURITY_FLAG + " " + POLICY_FLAG;
static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+";
static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*";
static final String MULTI_COMMAND_REGEX =
"(?s).*(" + //command read as single line
"(&[^>]|&&)|(\\|{1,2})|(\\|&)|" + //Matches '&','&&','|','||' and '|&'
"(`[^`]+`)|(\\$\\([^)]+\\))|" + //Matches occurrences of $() or ``
"(;)" + //Matches end of statement ';'
").*";
static final String CLEAN_CMD_REGEX =
"(" + SECURITY_FLAG + ")|" +
"(" + STRIP_POLICY_FLAG + ")";
static final String FILE_PERMISSION_FORMAT = " permission "
+ FilePermission.class.getCanonicalName()
+ " \"%1$s" + SEPARATOR + "-\", \"%2$s\";%n";
static final String HADOOP_HOME_PERMISSION = "%ngrant codeBase \"file:"
+ Paths.get(System.getProperty(SYSPROP_HADOOP_HOME_DIR))
+ SEPARATOR + "-\" {%n" +
" permission " + AllPermission.class.getCanonicalName() + ";%n};%n";
static final Logger LOG =
LoggerFactory.getLogger(NMContainerPolicyUtils.class);
/**
* Write new policy file to policyOutStream which will include read access
* to localize resources. Optionally a default policyFilePath can be
* specified to append a custom policy implementation to the new policy file
* @param policyOutStream OutputStream pointing to java.policy file
* @param localDirs Container local directories
* @param resources List of local container resources
* @param conf YARN configuration
* @throws IOException - If policy file generation is unable to read the
* base policy file or if it is unable to create a new policy file.
*/
static void generatePolicyFile(OutputStream policyOutStream,
List<String> localDirs, List<String> groupPolicyPaths,
Map<org.apache.hadoop.fs.Path, List<String>> resources,
Configuration conf)
throws IOException {
String policyFilePath =
conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY);
String filePermissions =
conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS,
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS);
Set<String> cacheDirs = new HashSet<>();
for(org.apache.hadoop.fs.Path path : resources.keySet()) {
cacheDirs.add(path.getParent().toString());
}
if (groupPolicyPaths != null) {
for(String policyPath : groupPolicyPaths) {
Files.copy(Paths.get(policyPath), policyOutStream);
}
} else if (policyFilePath == null) {
IOUtils.copyBytes(
NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
policyOutStream, conf, false);
} else {
Files.copy(Paths.get(policyFilePath), policyOutStream);
}
Formatter filePermissionFormat = new Formatter(policyOutStream,
StandardCharsets.UTF_8.name());
filePermissionFormat.format(HADOOP_HOME_PERMISSION);
filePermissionFormat.format("grant {%n");
for(String localDir : localDirs) {
filePermissionFormat.format(
FILE_PERMISSION_FORMAT, localDir, filePermissions);
}
for(String cacheDir : cacheDirs) {
filePermissionFormat.format(
FILE_PERMISSION_FORMAT, cacheDir, filePermissions);
}
filePermissionFormat.format("};%n");
filePermissionFormat.flush();
}
/**
* Modify command to enable the Java Security Manager and specify
* java.policy file. Will modify the passed commands to strip any
* existing java security configurations. Expects a java command to be the
* first and only executable provided in enforcing mode. In passive mode
* any commands with '||' or '&&' will not be modified.
* @param commands List of container commands
* @param env Container environment variables
* @param policyPath Path to the container specific policy file
* @param sandboxMode (enforcing, permissive, disabled) Determines
* whether non-java containers will be launched
* @throws ContainerExecutionException - Exception thrown if
* JVM Sandbox enabled in 'enforcing' mode and a non-java command is
* provided in the list of commands
*/
static void appendSecurityFlags(List<String> commands,
Map<String, String> env, Path policyPath, SandboxMode sandboxMode)
throws ContainerExecutionException {
for(int i = 0; i < commands.size(); i++){
String command = commands.get(i);
if(validateJavaHome(env.get(JAVA_HOME.name()))
&& command.matches(CONTAINS_JAVA_CMD)
&& !command.matches(MULTI_COMMAND_REGEX)){
command = command.replaceAll(CLEAN_CMD_REGEX, "");
String securityString = JVM_SECURITY_CMD + policyPath + " ";
if(LOG.isDebugEnabled()) {
securityString += SECURITY_DEBUG;
}
commands.set(i, command.replaceFirst(JAVA_CMD, securityString));
} else if (sandboxMode == SandboxMode.enforcing){
throw new ContainerExecutionException(
"Only JVM containers permitted in YARN sandbox mode (enforcing). "
+ "The following command can not be executed securely: " + command);
} else if (sandboxMode == SandboxMode.permissive){
LOG.warn("The container will run without the java security manager"
+ " due to an unsupported container command. The command"
+ " will be permitted to run in Sandbox permissive mode: "
+ command);
}
}
}
private static boolean validateJavaHome(String containerJavaHome)
throws ContainerExecutionException{
if (System.getenv(JAVA_HOME.name()) == null) {
throw new ContainerExecutionException(
"JAVA_HOME is not set for NodeManager");
}
if (containerJavaHome == null) {
throw new ContainerExecutionException(
"JAVA_HOME is not set for container");
}
return System.getenv(JAVA_HOME.name()).equals(containerJavaHome);
}
}
}
相关信息
相关文章
hadoop DefaultLinuxContainerRuntime 源码
hadoop DelegatingLinuxContainerRuntime 源码
hadoop DockerLinuxContainerRuntime 源码
hadoop LinuxContainerRuntime 源码
0
赞
热门推荐
-
2、 - 优质文章
-
3、 gate.io
-
7、 golang
-
9、 openharmony
-
10、 Vue中input框自动聚焦