hadoop PBHelperClient 源码
haddop PBHelperClient 代码
文件路径：/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.protocolPB;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader;
import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
import org.apache.hadoop.thirdparty.com.google.common.primitives.Shorts;
import org.apache.hadoop.thirdparty.protobuf.ByteString;
import org.apache.hadoop.thirdparty.protobuf.CodedInputStream;
import org.apache.hadoop.crypto.CipherOption;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclEntryScope;
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.inotify.Event;
import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockChecksumOptions;
import org.apache.hadoop.hdfs.protocol.BlockChecksumType;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.BlockType;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolStats;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.FsPermissionProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockFlagProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveStatsProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheFlagProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolStatsProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeStorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsECBlockGroupStatsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsReplicatedBlockStatsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.OpenFilesBatchResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.OpenFilesTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeActionProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmIdProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.EncryptionZoneProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ReencryptActionProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ReencryptionStateProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ZoneReencryptionStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AccessModeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AddErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenSecretProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ContentSummaryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CorruptFileBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CryptoProtocolVersionProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DataEncryptionKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto.AdminState;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfosProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeLocalInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto.StorageState;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RollingUpgradeStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportListingEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ZoneEncryptionInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto.XAttrNamespaceProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrSetFlagProto;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.CodecProto;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ChunkedArrayList;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.LimitInputStream;
import org.apache.hadoop.util.Lists;
/**
* Utilities for converting protobuf classes to and from hdfs-client side
* implementation classes and other helper utilities to help in dealing with
* protobuf.
*
* Note that when converting from an internal type to protobuf type, the
* converter never return null for protobuf type. The check for internal type
* being null must be done before calling the convert() method.
*/
public class PBHelperClient {
private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
XAttr.NameSpace.values();
private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES =
AclEntryType.values();
private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES =
AclEntryScope.values();
private static final FsAction[] FSACTION_VALUES =
FsAction.values();
private static ByteString getFixedByteString(String key) {
return ProtobufHelper.getFixedByteString(key);
}
/**
* Guava cache for caching String to ByteString encoding. Use this when the
* set of Strings is large, mutable, or unknown.
*/
private static LoadingCache<String, ByteString> bytestringCache =
CacheBuilder.newBuilder()
.maximumSize(10000)
.build(
new CacheLoader<String, ByteString>() {
public ByteString load(String key) {
return ByteString.copyFromUtf8(key);
}
});
private PBHelperClient() {
/** Hidden constructor */
}
public static ByteString getByteString(byte[] bytes) {
// return singleton to reduce object allocation
return ProtobufHelper.getByteString(bytes);
}
public static ShmId convert(ShortCircuitShmIdProto shmId) {
return new ShmId(shmId.getHi(), shmId.getLo());
}
public static DataChecksum.Type convert(HdfsProtos.ChecksumTypeProto type) {
return DataChecksum.Type.valueOf(type.getNumber());
}
public static HdfsProtos.ChecksumTypeProto convert(DataChecksum.Type type) {
return HdfsProtos.ChecksumTypeProto.forNumber(type.id);
}
public static HdfsProtos.BlockChecksumTypeProto convert(
BlockChecksumType type) {
switch(type) {
case MD5CRC:
return HdfsProtos.BlockChecksumTypeProto.MD5CRC;
case COMPOSITE_CRC:
return HdfsProtos.BlockChecksumTypeProto.COMPOSITE_CRC;
default:
throw new IllegalStateException(
"BUG: BlockChecksumType not found, type=" + type);
}
}
public static BlockChecksumType convert(
HdfsProtos.BlockChecksumTypeProto blockChecksumTypeProto) {
switch(blockChecksumTypeProto) {
case MD5CRC:
return BlockChecksumType.MD5CRC;
case COMPOSITE_CRC:
return BlockChecksumType.COMPOSITE_CRC;
default:
throw new IllegalStateException(
"BUG: BlockChecksumTypeProto not found, type="
+ blockChecksumTypeProto);
}
}
public static HdfsProtos.BlockChecksumOptionsProto convert(
BlockChecksumOptions options) {
return HdfsProtos.BlockChecksumOptionsProto.newBuilder()
.setBlockChecksumType(convert(options.getBlockChecksumType()))
.setStripeLength(options.getStripeLength())
.build();
}
public static BlockChecksumOptions convert(
HdfsProtos.BlockChecksumOptionsProto options) {
return new BlockChecksumOptions(
convert(options.getBlockChecksumType()),
options.getStripeLength());
}
public static ExtendedBlockProto convert(final ExtendedBlock b) {
if (b == null) return null;
return ExtendedBlockProto.newBuilder().
setPoolIdBytes(getFixedByteString(b.getBlockPoolId())).
setBlockId(b.getBlockId()).
setNumBytes(b.getNumBytes()).
setGenerationStamp(b.getGenerationStamp()).
build();
}
public static TokenProto convert(Token<?> tok) {
return ProtobufHelper.protoFromToken(tok);
}
public static ShortCircuitShmIdProto convert(ShmId shmId) {
return ShortCircuitShmIdProto.newBuilder().
setHi(shmId.getHi()).
setLo(shmId.getLo()).
build();
}
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
return ShortCircuitShmSlotProto.newBuilder().
setShmId(convert(slotId.getShmId())).
setSlotIdx(slotId.getSlotIdx()).
build();
}
public static DatanodeIDProto convert(DatanodeID dn) {
// For wire compatibility with older versions we transmit the StorageID
// which is the same as the DatanodeUuid. Since StorageID is a required
// field we pass the empty string if the DatanodeUuid is not yet known.
return DatanodeIDProto.newBuilder()
.setIpAddrBytes(dn.getIpAddrBytes())
.setHostNameBytes(dn.getHostNameBytes())
.setXferPort(dn.getXferPort())
.setDatanodeUuidBytes(dn.getDatanodeUuidBytes())
.setInfoPort(dn.getInfoPort())
.setInfoSecurePort(dn.getInfoSecurePort())
.setIpcPort(dn.getIpcPort()).build();
}
public static DatanodeInfoProto.AdminState convert(
final DatanodeInfo.AdminStates inAs) {
switch (inAs) {
case NORMAL: return DatanodeInfoProto.AdminState.NORMAL;
case DECOMMISSION_INPROGRESS:
return DatanodeInfoProto.AdminState.DECOMMISSION_INPROGRESS;
case DECOMMISSIONED: return DatanodeInfoProto.AdminState.DECOMMISSIONED;
case ENTERING_MAINTENANCE:
return DatanodeInfoProto.AdminState.ENTERING_MAINTENANCE;
case IN_MAINTENANCE:
return DatanodeInfoProto.AdminState.IN_MAINTENANCE;
default: return DatanodeInfoProto.AdminState.NORMAL;
}
}
public static DatanodeInfoProto convert(DatanodeInfo info) {
DatanodeInfoProto.Builder builder = DatanodeInfoProto.newBuilder();
if (info.getNetworkLocation() != null) {
builder.setLocationBytes(
bytestringCache.getUnchecked(info.getNetworkLocation()));
}
if (info.getUpgradeDomain() != null) {
builder.setUpgradeDomain(info.getUpgradeDomain());
}
builder
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
.setDfsUsed(info.getDfsUsed())
.setNonDfsUsed(info.getNonDfsUsed())
.setRemaining(info.getRemaining())
.setBlockPoolUsed(info.getBlockPoolUsed())
.setCacheCapacity(info.getCacheCapacity())
.setCacheUsed(info.getCacheUsed())
.setLastUpdate(info.getLastUpdate())
.setLastUpdateMonotonic(info.getLastUpdateMonotonic())
.setXceiverCount(info.getXceiverCount())
.setAdminState(convert(info.getAdminState()))
.setLastBlockReportTime(info.getLastBlockReportTime())
.setLastBlockReportMonotonic(info.getLastBlockReportMonotonic())
.setNumBlocks(info.getNumBlocks())
.build();
return builder.build();
}
public static List<? extends HdfsProtos.DatanodeInfoProto> convert(
DatanodeInfo[] dnInfos) {
return convert(dnInfos, 0);
}
/**
* Copy from {@code dnInfos} to a target of list of same size starting at
* {@code startIdx}.
*/
public static List<? extends HdfsProtos.DatanodeInfoProto> convert(
DatanodeInfo[] dnInfos, int startIdx) {
if (dnInfos == null)
return null;
ArrayList<HdfsProtos.DatanodeInfoProto> protos = Lists
.newArrayListWithCapacity(dnInfos.length);
for (int i = startIdx; i < dnInfos.length; i++) {
protos.add(convert(dnInfos[i]));
}
return protos;
}
public static List<Boolean> convert(boolean[] targetPinnings, int idx) {
List<Boolean> pinnings = new ArrayList<>();
if (targetPinnings == null) {
pinnings.add(Boolean.FALSE);
} else {
for (; idx < targetPinnings.length; ++idx) {
pinnings.add(targetPinnings[idx]);
}
}
return pinnings;
}
public static List<String> convert(String[] targetIds, int idx) {
List<String> ids = new ArrayList<>();
if (targetIds != null) {
for (; idx < targetIds.length; ++idx) {
ids.add(targetIds[idx]);
}
}
return ids;
}
public static ExtendedBlock convert(ExtendedBlockProto eb) {
if (eb == null) return null;
return new ExtendedBlock( eb.getPoolId(), eb.getBlockId(), eb.getNumBytes(),
eb.getGenerationStamp());
}
public static DatanodeLocalInfo convert(DatanodeLocalInfoProto proto) {
return new DatanodeLocalInfo(proto.getSoftwareVersion(),
proto.getConfigVersion(), proto.getUptime());
}
static public DatanodeInfoProto convertDatanodeInfo(DatanodeInfo di) {
if (di == null) return null;
return convert(di);
}
public static StorageTypeProto convertStorageType(StorageType type) {
switch(type) {
case DISK:
return StorageTypeProto.DISK;
case SSD:
return StorageTypeProto.SSD;
case ARCHIVE:
return StorageTypeProto.ARCHIVE;
case RAM_DISK:
return StorageTypeProto.RAM_DISK;
case PROVIDED:
return StorageTypeProto.PROVIDED;
case NVDIMM:
return StorageTypeProto.NVDIMM;
default:
throw new IllegalStateException(
"BUG: StorageType not found, type=" + type);
}
}
public static StorageType convertStorageType(StorageTypeProto type) {
switch(type) {
case DISK:
return StorageType.DISK;
case SSD:
return StorageType.SSD;
case ARCHIVE:
return StorageType.ARCHIVE;
case RAM_DISK:
return StorageType.RAM_DISK;
case PROVIDED:
return StorageType.PROVIDED;
case NVDIMM:
return StorageType.NVDIMM;
default:
throw new IllegalStateException(
"BUG: StorageTypeProto not found, type=" + type);
}
}
public static List<StorageTypeProto> convertStorageTypes(
StorageType[] types) {
return convertStorageTypes(types, 0);
}
public static List<StorageTypeProto> convertStorageTypes(
StorageType[] types, int startIdx) {
if (types == null) {
return null;
}
final List<StorageTypeProto> protos = new ArrayList<>(
types.length);
for (int i = startIdx; i < types.length; ++i) {
protos.add(convertStorageType(types[i]));
}
return protos;
}
public static InputStream vintPrefixed(final InputStream input)
throws IOException {
final int firstByte = input.read();
if (firstByte == -1) {
throw new EOFException(
"Unexpected EOF while trying to read response from server");
}
int size = CodedInputStream.readRawVarint32(firstByte, input);
assert size >= 0;
return new LimitInputStream(input, size);
}
public static CipherOption convert(HdfsProtos.CipherOptionProto proto) {
if (proto != null) {
CipherSuite suite = null;
if (proto.getSuite() != null) {
suite = convert(proto.getSuite());
}
byte[] inKey = null;
if (proto.getInKey() != null) {
inKey = proto.getInKey().toByteArray();
}
byte[] inIv = null;
if (proto.getInIv() != null) {
inIv = proto.getInIv().toByteArray();
}
byte[] outKey = null;
if (proto.getOutKey() != null) {
outKey = proto.getOutKey().toByteArray();
}
byte[] outIv = null;
if (proto.getOutIv() != null) {
outIv = proto.getOutIv().toByteArray();
}
return new CipherOption(suite, inKey, inIv, outKey, outIv);
}
return null;
}
public static CipherSuite convert(HdfsProtos.CipherSuiteProto proto) {
switch (proto) {
case AES_CTR_NOPADDING:
return CipherSuite.AES_CTR_NOPADDING;
case SM4_CTR_NOPADDING:
return CipherSuite.SM4_CTR_NOPADDING;
default:
// Set to UNKNOWN and stash the unknown enum value
CipherSuite suite = CipherSuite.UNKNOWN;
suite.setUnknownValue(proto.getNumber());
return suite;
}
}
public static HdfsProtos.CipherOptionProto convert(CipherOption option) {
if (option != null) {
HdfsProtos.CipherOptionProto.Builder builder =
HdfsProtos.CipherOptionProto.newBuilder();
if (option.getCipherSuite() != null) {
builder.setSuite(convert(option.getCipherSuite()));
}
if (option.getInKey() != null) {
builder.setInKey(getByteString(option.getInKey()));
}
if (option.getInIv() != null) {
builder.setInIv(getByteString(option.getInIv()));
}
if (option.getOutKey() != null) {
builder.setOutKey(getByteString(option.getOutKey()));
}
if (option.getOutIv() != null) {
builder.setOutIv(getByteString(option.getOutIv()));
}
return builder.build();
}
return null;
}
public static HdfsProtos.CipherSuiteProto convert(CipherSuite suite) {
switch (suite) {
case UNKNOWN:
return HdfsProtos.CipherSuiteProto.UNKNOWN;
case AES_CTR_NOPADDING:
return HdfsProtos.CipherSuiteProto.AES_CTR_NOPADDING;
case SM4_CTR_NOPADDING:
return HdfsProtos.CipherSuiteProto.SM4_CTR_NOPADDING;
default:
return null;
}
}
public static List<HdfsProtos.CipherOptionProto> convertCipherOptions(
List<CipherOption> options) {
if (options != null) {
List<HdfsProtos.CipherOptionProto> protos =
Lists.newArrayListWithCapacity(options.size());
for (CipherOption option : options) {
protos.add(convert(option));
}
return protos;
}
return null;
}
public static List<CipherOption> convertCipherOptionProtos(
List<HdfsProtos.CipherOptionProto> protos) {
if (protos != null) {
List<CipherOption> options =
Lists.newArrayListWithCapacity(protos.size());
for (HdfsProtos.CipherOptionProto proto : protos) {
options.add(convert(proto));
}
return options;
}
return null;
}
public static LocatedBlock convertLocatedBlockProto(LocatedBlockProto proto) {
if (proto == null) return null;
List<DatanodeInfoProto> locs = proto.getLocsList();
DatanodeInfo[] targets = new DatanodeInfo[locs.size()];
for (int i = 0; i < locs.size(); i++) {
targets[i] = convert(locs.get(i));
}
final StorageType[] storageTypes = convertStorageTypes(
proto.getStorageTypesList(), locs.size());
final int storageIDsCount = proto.getStorageIDsCount();
final String[] storageIDs;
if (storageIDsCount == 0) {
storageIDs = null;
} else {
Preconditions.checkState(storageIDsCount == locs.size());
storageIDs = proto.getStorageIDsList()
.toArray(new String[storageIDsCount]);
}
byte[] indices = null;
if (proto.hasBlockIndices()) {
indices = proto.getBlockIndices().toByteArray();
}
// Set values from the isCached list, re-using references from loc
List<DatanodeInfo> cachedLocs = new ArrayList<>(locs.size());
List<Boolean> isCachedList = proto.getIsCachedList();
for (int i=0; i<isCachedList.size(); i++) {
if (isCachedList.get(i)) {
cachedLocs.add(targets[i]);
}
}
final LocatedBlock lb;
if (indices == null) {
lb = new LocatedBlock(PBHelperClient.convert(proto.getB()), targets,
storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt(),
cachedLocs.toArray(new DatanodeInfo[cachedLocs.size()]));
} else {
lb = new LocatedStripedBlock(PBHelperClient.convert(proto.getB()),
targets, storageIDs, storageTypes, indices, proto.getOffset(),
proto.getCorrupt(),
cachedLocs.toArray(new DatanodeInfo[cachedLocs.size()]));
List<TokenProto> tokenProtos = proto.getBlockTokensList();
Token<BlockTokenIdentifier>[] blockTokens =
convertTokens(tokenProtos);
((LocatedStripedBlock) lb).setBlockTokens(blockTokens);
}
lb.setBlockToken(convert(proto.getBlockToken()));
return lb;
}
static public Token<BlockTokenIdentifier>[] convertTokens(
List<TokenProto> tokenProtos) {
@SuppressWarnings("unchecked")
Token<BlockTokenIdentifier>[] blockTokens = new Token[tokenProtos.size()];
for (int i = 0; i < blockTokens.length; i++) {
blockTokens[i] = convert(tokenProtos.get(i));
}
return blockTokens;
}
public static AccessModeProto convert(BlockTokenIdentifier.AccessMode aMode) {
switch (aMode) {
case READ: return AccessModeProto.READ;
case WRITE: return AccessModeProto.WRITE;
case COPY: return AccessModeProto.COPY;
case REPLACE: return AccessModeProto.REPLACE;
default:
throw new IllegalArgumentException("Unexpected AccessMode: " + aMode);
}
}
public static BlockTokenIdentifier.AccessMode convert(
AccessModeProto accessModeProto) {
switch (accessModeProto) {
case READ: return BlockTokenIdentifier.AccessMode.READ;
case WRITE: return BlockTokenIdentifier.AccessMode.WRITE;
case COPY: return BlockTokenIdentifier.AccessMode.COPY;
case REPLACE: return BlockTokenIdentifier.AccessMode.REPLACE;
default:
throw new IllegalArgumentException("Unexpected AccessModeProto: " +
accessModeProto);
}
}
public static BlockTokenSecretProto convert(
BlockTokenIdentifier blockTokenSecret) {
BlockTokenSecretProto.Builder builder =
BlockTokenSecretProto.newBuilder();
builder.setExpiryDate(blockTokenSecret.getExpiryDate());
builder.setKeyId(blockTokenSecret.getKeyId());
String userId = blockTokenSecret.getUserId();
if (userId != null) {
builder.setUserId(userId);
}
String blockPoolId = blockTokenSecret.getBlockPoolId();
if (blockPoolId != null) {
builder.setBlockPoolId(blockPoolId);
}
builder.setBlockId(blockTokenSecret.getBlockId());
for (BlockTokenIdentifier.AccessMode aMode :
blockTokenSecret.getAccessModes()) {
builder.addModes(convert(aMode));
}
for (StorageType storageType : blockTokenSecret.getStorageTypes()) {
builder.addStorageTypes(convertStorageType(storageType));
}
for (String storageId : blockTokenSecret.getStorageIds()) {
builder.addStorageIds(storageId);
}
byte[] handshake = blockTokenSecret.getHandshakeMsg();
if (handshake != null && handshake.length > 0) {
builder.setHandshakeSecret(getByteString(handshake));
}
return builder.build();
}
static public DatanodeInfo convert(DatanodeInfoProto di) {
if (di == null) {
return null;
}
DatanodeInfoBuilder dinfo =
new DatanodeInfoBuilder().setNodeID(convert(di.getId()))
.setNetworkLocation(di.hasLocation() ? di.getLocation() : null)
.setCapacity(di.getCapacity()).setDfsUsed(di.getDfsUsed())
.setRemaining(di.getRemaining())
.setBlockPoolUsed(di.getBlockPoolUsed())
.setCacheCapacity(di.getCacheCapacity())
.setCacheUsed(di.getCacheUsed()).setLastUpdate(di.getLastUpdate())
.setLastUpdateMonotonic(di.getLastUpdateMonotonic())
.setXceiverCount(di.getXceiverCount())
.setAdminState(convert(di.getAdminState())).setUpgradeDomain(
di.hasUpgradeDomain() ? di.getUpgradeDomain() : null)
.setLastBlockReportTime(di.hasLastBlockReportTime() ?
di.getLastBlockReportTime() : 0)
.setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ?
di.getLastBlockReportMonotonic() : 0)
.setNumBlocks(di.getNumBlocks());
if (di.hasNonDfsUsed()) {
dinfo.setNonDfsUsed(di.getNonDfsUsed());
} else {
// use the legacy way for older datanodes
long nonDFSUsed = di.getCapacity() - di.getDfsUsed() - di.getRemaining();
dinfo.setNonDfsUsed(nonDFSUsed < 0 ? 0 : nonDFSUsed);
}
return dinfo.build();
}
public static StorageType[] convertStorageTypes(
List<StorageTypeProto> storageTypesList, int expectedSize) {
final StorageType[] storageTypes = new StorageType[expectedSize];
if (storageTypesList.size() != expectedSize) {
// missing storage types
Preconditions.checkState(storageTypesList.isEmpty());
Arrays.fill(storageTypes, StorageType.DEFAULT);
} else {
for (int i = 0; i < storageTypes.length; ++i) {
storageTypes[i] = convertStorageType(storageTypesList.get(i));
}
}
return storageTypes;
}
public static Token<BlockTokenIdentifier> convert(
TokenProto blockToken) {
return (Token<BlockTokenIdentifier>) ProtobufHelper
.tokenFromProto(blockToken);
}
// DatanodeId
public static DatanodeID convert(DatanodeIDProto dn) {
return new DatanodeID(dn.getIpAddr(), dn.getHostName(),
dn.getDatanodeUuid(), dn.getXferPort(), dn.getInfoPort(),
dn.hasInfoSecurePort() ? dn.getInfoSecurePort() : 0, dn.getIpcPort());
}
public static AdminStates convert(AdminState adminState) {
switch(adminState) {
case DECOMMISSION_INPROGRESS:
return AdminStates.DECOMMISSION_INPROGRESS;
case DECOMMISSIONED:
return AdminStates.DECOMMISSIONED;
case ENTERING_MAINTENANCE:
return AdminStates.ENTERING_MAINTENANCE;
case IN_MAINTENANCE:
return AdminStates.IN_MAINTENANCE;
case NORMAL:
default:
return AdminStates.NORMAL;
}
}
// LocatedBlocks
public static LocatedBlocks convert(LocatedBlocksProto lb) {
return new LocatedBlocks(
lb.getFileLength(), lb.getUnderConstruction(),
convertLocatedBlocks(lb.getBlocksList()),
lb.hasLastBlock() ?
convertLocatedBlockProto(lb.getLastBlock()) : null,
lb.getIsLastBlockComplete(),
lb.hasFileEncryptionInfo() ? convert(lb.getFileEncryptionInfo()) : null,
lb.hasEcPolicy() ? convertErasureCodingPolicy(lb.getEcPolicy()) : null);
}
public static BlockStoragePolicy[] convertStoragePolicies(
List<BlockStoragePolicyProto> policyProtos) {
if (policyProtos == null || policyProtos.size() == 0) {
return new BlockStoragePolicy[0];
}
BlockStoragePolicy[] policies = new BlockStoragePolicy[policyProtos.size()];
int i = 0;
for (BlockStoragePolicyProto proto : policyProtos) {
policies[i++] = convert(proto);
}
return policies;
}
public static EventBatchList convert(GetEditsFromTxidResponseProto resp)
throws IOException {
final InotifyProtos.EventsListProto list = resp.getEventsList();
final long firstTxid = list.getFirstTxid();
final long lastTxid = list.getLastTxid();
List<EventBatch> batches = Lists.newArrayList();
if (list.getEventsList().size() > 0) {
throw new IOException("Can't handle old inotify server response.");
}
for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
long txid = bp.getTxid();
if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
throw new IOException("Error converting TxidResponseProto: got a " +
"transaction id " + txid + " that was outside the range of [" +
firstTxid + ", " + lastTxid + "].");
}
List<Event> events = Lists.newArrayList();
for (InotifyProtos.EventProto p : bp.getEventsList()) {
switch (p.getType()) {
case EVENT_CLOSE:
InotifyProtos.CloseEventProto close =
InotifyProtos.CloseEventProto.parseFrom(p.getContents());
events.add(new Event.CloseEvent(close.getPath(),
close.getFileSize(), close.getTimestamp()));
break;
case EVENT_CREATE:
InotifyProtos.CreateEventProto create =
InotifyProtos.CreateEventProto.parseFrom(p.getContents());
Event.CreateEvent.Builder builder = new Event.CreateEvent.Builder()
.iNodeType(createTypeConvert(create.getType()))
.path(create.getPath())
.ctime(create.getCtime())
.ownerName(create.getOwnerName())
.groupName(create.getGroupName())
.perms(convert(create.getPerms()))
.replication(create.getReplication())
.symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
create.getSymlinkTarget())
.defaultBlockSize(create.getDefaultBlockSize())
.overwrite(create.getOverwrite());
if (create.hasErasureCoded()) {
builder.erasureCoded(create.getErasureCoded());
}
events.add(builder.build());
break;
case EVENT_METADATA:
InotifyProtos.MetadataUpdateEventProto meta =
InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
events.add(new Event.MetadataUpdateEvent.Builder()
.path(meta.getPath())
.metadataType(metadataUpdateTypeConvert(meta.getType()))
.mtime(meta.getMtime())
.atime(meta.getAtime())
.replication(meta.getReplication())
.ownerName(
meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
.groupName(
meta.getGroupName().isEmpty() ? null : meta.getGroupName())
.perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
.acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
meta.getAclsList()))
.xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
meta.getXAttrsList()))
.xAttrsRemoved(meta.getXAttrsRemoved())
.build());
break;
case EVENT_RENAME:
InotifyProtos.RenameEventProto rename =
InotifyProtos.RenameEventProto.parseFrom(p.getContents());
events.add(new Event.RenameEvent.Builder()
.srcPath(rename.getSrcPath())
.dstPath(rename.getDestPath())
.timestamp(rename.getTimestamp())
.build());
break;
case EVENT_APPEND:
InotifyProtos.AppendEventProto append =
InotifyProtos.AppendEventProto.parseFrom(p.getContents());
events.add(new Event.AppendEvent.Builder().path(append.getPath())
.newBlock(append.hasNewBlock() && append.getNewBlock())
.build());
break;
case EVENT_UNLINK:
InotifyProtos.UnlinkEventProto unlink =
InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
events.add(new Event.UnlinkEvent.Builder()
.path(unlink.getPath())
.timestamp(unlink.getTimestamp())
.build());
break;
case EVENT_TRUNCATE:
InotifyProtos.TruncateEventProto truncate =
InotifyProtos.TruncateEventProto.parseFrom(p.getContents());
events.add(new Event.TruncateEvent(truncate.getPath(),
truncate.getFileSize(), truncate.getTimestamp()));
break;
default:
throw new RuntimeException("Unexpected inotify event type: " +
p.getType());
}
}
batches.add(new EventBatch(txid, events.toArray(new Event[0])));
}
return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}
// Located Block Arrays and Lists
public static LocatedBlockProto[] convertLocatedBlocks(LocatedBlock[] lb) {
if (lb == null) return null;
return convertLocatedBlocks2(Arrays.asList(lb))
.toArray(new LocatedBlockProto[lb.length]);
}
public static LocatedBlock[] convertLocatedBlocks(LocatedBlockProto[] lb) {
if (lb == null) return null;
return convertLocatedBlocks(Arrays.asList(lb))
.toArray(new LocatedBlock[lb.length]);
}
public static List<LocatedBlock> convertLocatedBlocks(
List<LocatedBlockProto> lb) {
if (lb == null) return null;
final int len = lb.size();
List<LocatedBlock> result = new ArrayList<>(len);
for (LocatedBlockProto aLb : lb) {
result.add(convertLocatedBlockProto(aLb));
}
return result;
}
public static List<LocatedBlockProto> convertLocatedBlocks2(
List<LocatedBlock> lb) {
if (lb == null) return null;
final int len = lb.size();
List<LocatedBlockProto> result = new ArrayList<>(len);
for (LocatedBlock aLb : lb) {
result.add(convertLocatedBlock(aLb));
}
return result;
}
public static LocatedBlockProto convertLocatedBlock(LocatedBlock b) {
if (b == null) return null;
Builder builder = LocatedBlockProto.newBuilder();
DatanodeInfo[] locs = b.getLocations();
List<DatanodeInfo> cachedLocs =
Lists.newLinkedList(Arrays.asList(b.getCachedLocations()));
for (int i = 0; i < locs.length; i++) {
DatanodeInfo loc = locs[i];
builder.addLocs(i, PBHelperClient.convert(loc));
boolean locIsCached = cachedLocs.contains(loc);
builder.addIsCached(locIsCached);
if (locIsCached) {
cachedLocs.remove(loc);
}
}
Preconditions.checkArgument(cachedLocs.size() == 0,
"Found additional cached replica locations that are not in the set of"
+ " storage-backed locations!");
StorageType[] storageTypes = b.getStorageTypes();
if (storageTypes != null) {
for (StorageType storageType : storageTypes) {
builder.addStorageTypes(convertStorageType(storageType));
}
}
final String[] storageIDs = b.getStorageIDs();
if (storageIDs != null) {
builder.addAllStorageIDs(Arrays.asList(storageIDs));
}
if (b instanceof LocatedStripedBlock) {
LocatedStripedBlock sb = (LocatedStripedBlock) b;
byte[] indices = sb.getBlockIndices();
builder.setBlockIndices(PBHelperClient.getByteString(indices));
Token<BlockTokenIdentifier>[] blockTokens = sb.getBlockTokens();
builder.addAllBlockTokens(convert(blockTokens));
}
return builder.setB(PBHelperClient.convert(b.getBlock()))
.setBlockToken(PBHelperClient.convert(b.getBlockToken()))
.setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build();
}
public static List<TokenProto> convert(
Token<BlockTokenIdentifier>[] blockTokens) {
List<TokenProto> results = new ArrayList<>(blockTokens.length);
for (Token<BlockTokenIdentifier> bt : blockTokens) {
results.add(convert(bt));
}
return results;
}
public static List<Integer> convertBlockIndices(byte[] blockIndices) {
List<Integer> results = new ArrayList<>(blockIndices.length);
for (byte bt : blockIndices) {
results.add(Integer.valueOf(bt));
}
return results;
}
public static byte[] convertBlockIndices(List<Integer> blockIndices) {
byte[] blkIndices = new byte[blockIndices.size()];
for (int i = 0; i < blockIndices.size(); i++) {
blkIndices[i] = (byte) blockIndices.get(i).intValue();
}
return blkIndices;
}
public static BlockStoragePolicy convert(BlockStoragePolicyProto proto) {
List<StorageTypeProto> cList = proto.getCreationPolicy()
.getStorageTypesList();
StorageType[] creationTypes = convertStorageTypes(cList, cList.size());
List<StorageTypeProto> cfList = proto.hasCreationFallbackPolicy() ? proto
.getCreationFallbackPolicy().getStorageTypesList() : null;
StorageType[] creationFallbackTypes = cfList == null ? StorageType
.EMPTY_ARRAY : convertStorageTypes(cfList, cfList.size());
List<StorageTypeProto> rfList = proto.hasReplicationFallbackPolicy() ?
proto.getReplicationFallbackPolicy().getStorageTypesList() : null;
StorageType[] replicationFallbackTypes = rfList == null ? StorageType
.EMPTY_ARRAY : convertStorageTypes(rfList, rfList.size());
return new BlockStoragePolicy((byte) proto.getPolicyId(), proto.getName(),
creationTypes, creationFallbackTypes, replicationFallbackTypes);
}
public static FsActionProto convert(FsAction v) {
return FsActionProto.forNumber(v != null ? v.ordinal() : 0);
}
public static XAttrProto convertXAttrProto(XAttr a) {
XAttrProto.Builder builder = XAttrProto.newBuilder();
builder.setNamespace(convert(a.getNameSpace()));
if (a.getName() != null) {
builder.setName(a.getName());
}
if (a.getValue() != null) {
builder.setValue(getByteString(a.getValue()));
}
return builder.build();
}
public static List<XAttr> convert(ListXAttrsResponseProto a) {
final List<XAttrProto> xAttrs = a.getXAttrsList();
return convertXAttrs(xAttrs);
}
public static List<XAttr> convert(GetXAttrsResponseProto a) {
List<XAttrProto> xAttrs = a.getXAttrsList();
return convertXAttrs(xAttrs);
}
public static List<XAttr> convertXAttrs(List<XAttrProto> xAttrSpec) {
ArrayList<XAttr> xAttrs = Lists.newArrayListWithCapacity(xAttrSpec.size());
for (XAttrProto a : xAttrSpec) {
XAttr.Builder builder = new XAttr.Builder();
builder.setNameSpace(convert(a.getNamespace()));
if (a.hasName()) {
builder.setName(a.getName());
}
if (a.hasValue()) {
builder.setValue(a.getValue().toByteArray());
}
xAttrs.add(builder.build());
}
return xAttrs;
}
static XAttrNamespaceProto convert(XAttr.NameSpace v) {
return XAttrNamespaceProto.forNumber(v.ordinal());
}
static XAttr.NameSpace convert(XAttrNamespaceProto v) {
return castEnum(v, XATTR_NAMESPACE_VALUES);
}
static <T extends Enum<T>, U extends Enum<U>> U castEnum(T from, U[] to) {
return to[from.ordinal()];
}
static InotifyProtos.MetadataUpdateType metadataUpdateTypeConvert(
Event.MetadataUpdateEvent.MetadataType type) {
switch (type) {
case TIMES:
return InotifyProtos.MetadataUpdateType.META_TYPE_TIMES;
case REPLICATION:
return InotifyProtos.MetadataUpdateType.META_TYPE_REPLICATION;
case OWNER:
return InotifyProtos.MetadataUpdateType.META_TYPE_OWNER;
case PERMS:
return InotifyProtos.MetadataUpdateType.META_TYPE_PERMS;
case ACLS:
return InotifyProtos.MetadataUpdateType.META_TYPE_ACLS;
case XATTRS:
return InotifyProtos.MetadataUpdateType.META_TYPE_XATTRS;
default:
return null;
}
}
private static Event.MetadataUpdateEvent.MetadataType metadataUpdateTypeConvert(
InotifyProtos.MetadataUpdateType type) {
switch (type) {
case META_TYPE_TIMES:
return Event.MetadataUpdateEvent.MetadataType.TIMES;
case META_TYPE_REPLICATION:
return Event.MetadataUpdateEvent.MetadataType.REPLICATION;
case META_TYPE_OWNER:
return Event.MetadataUpdateEvent.MetadataType.OWNER;
case META_TYPE_PERMS:
return Event.MetadataUpdateEvent.MetadataType.PERMS;
case META_TYPE_ACLS:
return Event.MetadataUpdateEvent.MetadataType.ACLS;
case META_TYPE_XATTRS:
return Event.MetadataUpdateEvent.MetadataType.XATTRS;
default:
return null;
}
}
static InotifyProtos.INodeType createTypeConvert(Event.CreateEvent.INodeType
type) {
switch (type) {
case DIRECTORY:
return InotifyProtos.INodeType.I_TYPE_DIRECTORY;
case FILE:
return InotifyProtos.INodeType.I_TYPE_FILE;
case SYMLINK:
return InotifyProtos.INodeType.I_TYPE_SYMLINK;
default:
return null;
}
}
public static List<LocatedBlock> convertLocatedBlock(
List<LocatedBlockProto> lb) {
if (lb == null) return null;
final int len = lb.size();
List<LocatedBlock> result = new ArrayList<>(len);
for (LocatedBlockProto aLb : lb) {
result.add(convertLocatedBlockProto(aLb));
}
return result;
}
public static List<AclEntry> convertAclEntry(List<AclEntryProto> aclSpec) {
ArrayList<AclEntry> r = Lists.newArrayListWithCapacity(aclSpec.size());
for (AclEntryProto e : aclSpec) {
AclEntry.Builder builder = new AclEntry.Builder();
builder.setType(convert(e.getType()));
builder.setScope(convert(e.getScope()));
builder.setPermission(convert(e.getPermissions()));
if (e.hasName()) {
builder.setName(e.getName());
}
r.add(builder.build());
}
return r;
}
static AclEntryScopeProto convert(AclEntryScope v) {
return AclEntryScopeProto.forNumber(v.ordinal());
}
private static AclEntryScope convert(AclEntryScopeProto v) {
return castEnum(v, ACL_ENTRY_SCOPE_VALUES);
}
static AclEntryTypeProto convert(AclEntryType e) {
return AclEntryTypeProto.forNumber(e.ordinal());
}
private static AclEntryType convert(AclEntryTypeProto v) {
return castEnum(v, ACL_ENTRY_TYPE_VALUES);
}
public static FsAction convert(FsActionProto v) {
return castEnum(v, FSACTION_VALUES);
}
public static FsPermission convert(FsPermissionProto p) {
return new FsPermission((short)p.getPerm());
}
private static Event.CreateEvent.INodeType createTypeConvert(
InotifyProtos.INodeType type) {
switch (type) {
case I_TYPE_DIRECTORY:
return Event.CreateEvent.INodeType.DIRECTORY;
case I_TYPE_FILE:
return Event.CreateEvent.INodeType.FILE;
case I_TYPE_SYMLINK:
return Event.CreateEvent.INodeType.SYMLINK;
default:
return null;
}
}
public static HdfsProtos.FileEncryptionInfoProto convert(
FileEncryptionInfo info) {
if (info == null) {
return null;
}
return HdfsProtos.FileEncryptionInfoProto.newBuilder()
.setSuite(convert(info.getCipherSuite()))
.setCryptoProtocolVersion(convert(info.getCryptoProtocolVersion()))
.setKey(getByteString(info.getEncryptedDataEncryptionKey()))
.setIv(getByteString(info.getIV()))
.setEzKeyVersionName(info.getEzKeyVersionName())
.setKeyName(info.getKeyName())
.build();
}
public static CryptoProtocolVersionProto convert(CryptoProtocolVersion
version) {
switch(version) {
case UNKNOWN:
return CryptoProtocolVersionProto.UNKNOWN_PROTOCOL_VERSION;
case ENCRYPTION_ZONES:
return CryptoProtocolVersionProto.ENCRYPTION_ZONES;
default:
return null;
}
}
public static FileEncryptionInfo convert(
HdfsProtos.FileEncryptionInfoProto proto) {
if (proto == null) {
return null;
}
CipherSuite suite = convert(proto.getSuite());
CryptoProtocolVersion version = convert(proto.getCryptoProtocolVersion());
byte[] key = proto.getKey().toByteArray();
byte[] iv = proto.getIv().toByteArray();
String ezKeyVersionName = proto.getEzKeyVersionName();
String keyName = proto.getKeyName();
return new FileEncryptionInfo(suite, version, key, iv, keyName,
ezKeyVersionName);
}
public static CryptoProtocolVersion convert(CryptoProtocolVersionProto
proto) {
switch(proto) {
case ENCRYPTION_ZONES:
return CryptoProtocolVersion.ENCRYPTION_ZONES;
default:
// Set to UNKNOWN and stash the unknown enum value
CryptoProtocolVersion version = CryptoProtocolVersion.UNKNOWN;
version.setUnknownValue(proto.getNumber());
return version;
}
}
public static List<XAttrProto> convertXAttrProto(
List<XAttr> xAttrSpec) {
if (xAttrSpec == null) {
return Lists.newArrayListWithCapacity(0);
}
ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity(
xAttrSpec.size());
for (XAttr a : xAttrSpec) {
XAttrProto.Builder builder = XAttrProto.newBuilder();
builder.setNamespace(convert(a.getNameSpace()));
if (a.getName() != null) {
builder.setName(a.getName());
}
if (a.getValue() != null) {
builder.setValue(getByteString(a.getValue()));
}
xAttrs.add(builder.build());
}
return xAttrs;
}
/**
* The flag field in PB is a bitmask whose values are the same a the
* emum values of XAttrSetFlag
*/
public static int convert(EnumSet<XAttrSetFlag> flag) {
int value = 0;
if (flag.contains(XAttrSetFlag.CREATE)) {
value |= XAttrSetFlagProto.XATTR_CREATE.getNumber();
}
if (flag.contains(XAttrSetFlag.REPLACE)) {
value |= XAttrSetFlagProto.XATTR_REPLACE.getNumber();
}
return value;
}
public static EncryptionZone convert(EncryptionZoneProto proto) {
return new EncryptionZone(proto.getId(), proto.getPath(),
convert(proto.getSuite()), convert(proto.getCryptoProtocolVersion()),
proto.getKeyName());
}
public static OpenFilesBatchResponseProto convert(OpenFileEntry
openFileEntry) {
return OpenFilesBatchResponseProto.newBuilder()
.setId(openFileEntry.getId())
.setPath(openFileEntry.getFilePath())
.setClientName(openFileEntry.getClientName())
.setClientMachine(openFileEntry.getClientMachine())
.build();
}
public static OpenFileEntry convert(OpenFilesBatchResponseProto proto) {
return new OpenFileEntry(proto.getId(), proto.getPath(),
proto.getClientName(), proto.getClientMachine());
}
public static AclStatus convert(GetAclStatusResponseProto e) {
AclStatusProto r = e.getResult();
AclStatus.Builder builder = new AclStatus.Builder();
builder.owner(r.getOwner()).group(r.getGroup()).stickyBit(r.getSticky())
.addEntries(convertAclEntry(r.getEntriesList()));
if (r.hasPermission()) {
builder.setPermission(convert(r.getPermission()));
}
return builder.build();
}
public static List<AclEntryProto> convertAclEntryProto(
List<AclEntry> aclSpec) {
ArrayList<AclEntryProto> r = Lists.newArrayListWithCapacity(aclSpec.size());
for (AclEntry e : aclSpec) {
AclEntryProto.Builder builder = AclEntryProto.newBuilder();
builder.setType(convert(e.getType()));
builder.setScope(convert(e.getScope()));
builder.setPermissions(convert(e.getPermission()));
if (e.getName() != null) {
builder.setName(e.getName());
}
r.add(builder.build());
}
return r;
}
public static CachePoolEntry convert(CachePoolEntryProto proto) {
CachePoolInfo info = convert(proto.getInfo());
CachePoolStats stats = convert(proto.getStats());
return new CachePoolEntry(info, stats);
}
public static CachePoolInfo convert (CachePoolInfoProto proto) {
// Pool name is a required field, the rest are optional
String poolName = Preconditions.checkNotNull(proto.getPoolName());
CachePoolInfo info = new CachePoolInfo(poolName);
if (proto.hasOwnerName()) {
info.setOwnerName(proto.getOwnerName());
}
if (proto.hasGroupName()) {
info.setGroupName(proto.getGroupName());
}
if (proto.hasMode()) {
info.setMode(new FsPermission((short)proto.getMode()));
}
if (proto.hasLimit()) {
info.setLimit(proto.getLimit());
}
if (proto.hasDefaultReplication()) {
info.setDefaultReplication(Shorts.checkedCast(
proto.getDefaultReplication()));
}
if (proto.hasMaxRelativeExpiry()) {
info.setMaxRelativeExpiryMs(proto.getMaxRelativeExpiry());
}
return info;
}
public static CachePoolStats convert (CachePoolStatsProto proto) {
CachePoolStats.Builder builder = new CachePoolStats.Builder();
builder.setBytesNeeded(proto.getBytesNeeded());
builder.setBytesCached(proto.getBytesCached());
builder.setBytesOverlimit(proto.getBytesOverlimit());
builder.setFilesNeeded(proto.getFilesNeeded());
builder.setFilesCached(proto.getFilesCached());
return builder.build();
}
public static CachePoolInfoProto convert(CachePoolInfo info) {
CachePoolInfoProto.Builder builder = CachePoolInfoProto.newBuilder();
builder.setPoolName(info.getPoolName());
if (info.getOwnerName() != null) {
builder.setOwnerName(info.getOwnerName());
}
if (info.getGroupName() != null) {
builder.setGroupName(info.getGroupName());
}
if (info.getMode() != null) {
builder.setMode(info.getMode().toShort());
}
if (info.getLimit() != null) {
builder.setLimit(info.getLimit());
}
if (info.getDefaultReplication() != null) {
builder.setDefaultReplication(info.getDefaultReplication());
}
if (info.getMaxRelativeExpiryMs() != null) {
builder.setMaxRelativeExpiry(info.getMaxRelativeExpiryMs());
}
return builder.build();
}
public static CacheDirectiveInfoProto convert(CacheDirectiveInfo info) {
CacheDirectiveInfoProto.Builder builder =
CacheDirectiveInfoProto.newBuilder();
if (info.getId() != null) {
builder.setId(info.getId());
}
if (info.getPath() != null) {
builder.setPath(info.getPath().toUri().getPath());
}
if (info.getReplication() != null) {
builder.setReplication(info.getReplication());
}
if (info.getPool() != null) {
builder.setPool(info.getPool());
}
if (info.getExpiration() != null) {
builder.setExpiration(convert(info.getExpiration()));
}
return builder.build();
}
public static CacheDirectiveInfoExpirationProto convert(
CacheDirectiveInfo.Expiration expiration) {
return CacheDirectiveInfoExpirationProto.newBuilder()
.setIsRelative(expiration.isRelative())
.setMillis(expiration.getMillis())
.build();
}
public static CacheDirectiveEntry convert(CacheDirectiveEntryProto proto) {
CacheDirectiveInfo info = convert(proto.getInfo());
CacheDirectiveStats stats = convert(proto.getStats());
return new CacheDirectiveEntry(info, stats);
}
public static CacheDirectiveStats convert(CacheDirectiveStatsProto proto) {
CacheDirectiveStats.Builder builder = new CacheDirectiveStats.Builder();
builder.setBytesNeeded(proto.getBytesNeeded());
builder.setBytesCached(proto.getBytesCached());
builder.setFilesNeeded(proto.getFilesNeeded());
builder.setFilesCached(proto.getFilesCached());
builder.setHasExpired(proto.getHasExpired());
return builder.build();
}
public static CacheDirectiveInfo convert(CacheDirectiveInfoProto proto) {
CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder();
if (proto.hasId()) {
builder.setId(proto.getId());
}
if (proto.hasPath()) {
builder.setPath(new Path(proto.getPath()));
}
if (proto.hasReplication()) {
builder.setReplication(Shorts.checkedCast(
proto.getReplication()));
}
if (proto.hasPool()) {
builder.setPool(proto.getPool());
}
if (proto.hasExpiration()) {
builder.setExpiration(convert(proto.getExpiration()));
}
return builder.build();
}
public static CacheDirectiveInfo.Expiration convert(
CacheDirectiveInfoExpirationProto proto) {
if (proto.getIsRelative()) {
return CacheDirectiveInfo.Expiration.newRelative(proto.getMillis());
}
return CacheDirectiveInfo.Expiration.newAbsolute(proto.getMillis());
}
public static int convertCacheFlags(EnumSet<CacheFlag> flags) {
int value = 0;
if (flags.contains(CacheFlag.FORCE)) {
value |= CacheFlagProto.FORCE.getNumber();
}
return value;
}
public static SnapshotDiffReport convert(
SnapshotDiffReportProto reportProto) {
if (reportProto == null) {
return null;
}
String snapshotDir = reportProto.getSnapshotRoot();
String fromSnapshot = reportProto.getFromSnapshot();
String toSnapshot = reportProto.getToSnapshot();
List<SnapshotDiffReportEntryProto> list = reportProto
.getDiffReportEntriesList();
List<DiffReportEntry> entries = new ChunkedArrayList<>();
for (SnapshotDiffReportEntryProto entryProto : list) {
DiffReportEntry entry = convert(entryProto);
if (entry != null)
entries.add(entry);
}
return new SnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot,
entries);
}
public static DiffReportEntry convert(SnapshotDiffReportEntryProto entry) {
if (entry == null) {
return null;
}
DiffType type = DiffType.getTypeFromLabel(entry
.getModificationLabel());
return type == null ? null : new DiffReportEntry(type, entry.getFullpath()
.toByteArray(), entry.hasTargetPath() ? entry.getTargetPath()
.toByteArray() : null);
}
public static SnapshotDiffReportListing convert(
SnapshotDiffReportListingProto reportProto) {
if (reportProto == null) {
return null;
}
List<SnapshotDiffReportListingEntryProto> modifyList =
reportProto.getModifiedEntriesList();
List<DiffReportListingEntry> modifiedEntries = new ChunkedArrayList<>();
for (SnapshotDiffReportListingEntryProto entryProto : modifyList) {
DiffReportListingEntry entry = convert(entryProto);
if (entry != null) {
modifiedEntries.add(entry);
}
}
List<SnapshotDiffReportListingEntryProto> createList =
reportProto.getCreatedEntriesList();
List<DiffReportListingEntry> createdEntries = new ChunkedArrayList<>();
for (SnapshotDiffReportListingEntryProto entryProto : createList) {
DiffReportListingEntry entry = convert(entryProto);
if (entry != null) {
createdEntries.add(entry);
}
}
List<SnapshotDiffReportListingEntryProto> deletedList =
reportProto.getDeletedEntriesList();
List<DiffReportListingEntry> deletedEntries = new ChunkedArrayList<>();
for (SnapshotDiffReportListingEntryProto entryProto : deletedList) {
DiffReportListingEntry entry = convert(entryProto);
if (entry != null) {
deletedEntries.add(entry);
}
}
byte[] startPath = reportProto.getCursor().getStartPath().toByteArray();
boolean isFromEarlier = reportProto.getIsFromEarlier();
int index = reportProto.getCursor().getIndex();
return new SnapshotDiffReportListing(startPath, modifiedEntries,
createdEntries, deletedEntries, index, isFromEarlier);
}
public static DiffReportListingEntry convert(
SnapshotDiffReportListingEntryProto entry) {
if (entry == null) {
return null;
}
long dirId = entry.getDirId();
long fileId = entry.getFileId();
boolean isReference = entry.getIsReference();
byte[] sourceName = entry.getFullpath().toByteArray();
byte[] targetName =
entry.hasTargetPath() ? entry.getTargetPath().toByteArray() : null;
return new DiffReportListingEntry(dirId, fileId, sourceName, isReference,
targetName);
}
public static SnapshottableDirectoryStatus[] convert(
SnapshottableDirectoryListingProto sdlp) {
if (sdlp == null)
return null;
List<SnapshottableDirectoryStatusProto> list = sdlp
.getSnapshottableDirListingList();
if (list.isEmpty()) {
return new SnapshottableDirectoryStatus[0];
} else {
SnapshottableDirectoryStatus[] result =
new SnapshottableDirectoryStatus[list.size()];
for (int i = 0; i < list.size(); i++) {
result[i] = convert(list.get(i));
}
return result;
}
}
public static SnapshottableDirectoryStatus convert(
SnapshottableDirectoryStatusProto sdirStatusProto) {
if (sdirStatusProto == null) {
return null;
}
final HdfsFileStatusProto status = sdirStatusProto.getDirStatus();
EnumSet<HdfsFileStatus.Flags> flags = status.hasFlags()
? convertFlags(status.getFlags())
: convertFlags(status.getPermission());
return new SnapshottableDirectoryStatus(
status.getModificationTime(),
status.getAccessTime(),
convert(status.getPermission()),
flags,
status.getOwner(),
status.getGroup(),
status.getPath().toByteArray(),
status.getFileId(),
status.getChildrenNum(),
sdirStatusProto.getSnapshotNumber(),
sdirStatusProto.getSnapshotQuota(),
sdirStatusProto.getParentFullpath().toByteArray());
}
public static SnapshotStatus[] convert(
HdfsProtos.SnapshotListingProto sdlp) {
if (sdlp == null) {
return null;
}
List<HdfsProtos.SnapshotStatusProto> list = sdlp
.getSnapshotListingList();
if (list.isEmpty()) {
return new SnapshotStatus[0];
} else {
SnapshotStatus[] result =
new SnapshotStatus[list.size()];
for (int i = 0; i < list.size(); i++) {
result[i] = convert(list.get(i));
}
return result;
}
}
public static SnapshotStatus convert(
HdfsProtos.SnapshotStatusProto sdirStatusProto) {
if (sdirStatusProto == null) {
return null;
}
final HdfsFileStatusProto status = sdirStatusProto.getDirStatus();
EnumSet<HdfsFileStatus.Flags> flags = status.hasFlags()
? convertFlags(status.getFlags())
: convertFlags(status.getPermission());
return new SnapshotStatus(
status.getModificationTime(),
status.getAccessTime(),
convert(status.getPermission()),
flags,
status.getOwner(),
status.getGroup(),
status.getPath().toByteArray(),
status.getFileId(),
status.getChildrenNum(),
sdirStatusProto.getSnapshotID(),
sdirStatusProto.getIsDeleted(),
sdirStatusProto.getParentFullpath().toByteArray());
}
// DataEncryptionKey
public static DataEncryptionKey convert(DataEncryptionKeyProto bet) {
String encryptionAlgorithm = bet.getEncryptionAlgorithm();
return new DataEncryptionKey(bet.getKeyId(),
bet.getBlockPoolId(),
bet.getNonce().toByteArray(),
bet.getEncryptionKey().toByteArray(),
bet.getExpiryDate(),
encryptionAlgorithm.isEmpty() ? null : encryptionAlgorithm);
}
public static Token<DelegationTokenIdentifier> convertDelegationToken(
TokenProto blockToken) {
return new Token<>(blockToken.getIdentifier()
.toByteArray(), blockToken.getPassword().toByteArray(), new Text(
blockToken.getKind()), new Text(blockToken.getService()));
}
// Arrays of DatanodeId
public static DatanodeIDProto[] convert(DatanodeID[] did) {
if (did == null)
return null;
final int len = did.length;
DatanodeIDProto[] result = new DatanodeIDProto[len];
for (int i = 0; i < len; ++i) {
result[i] = convert(did[i]);
}
return result;
}
public static FsPermissionProto convert(FsPermission p) {
return FsPermissionProto.newBuilder().setPerm(p.toShort()).build();
}
public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
if (fs == null) {
return null;
}
EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags()
? convertFlags(fs.getFlags())
: convertFlags(fs.getPermission());
HdfsFileStatus hdfsFileStatus = new HdfsFileStatus.Builder()
.length(fs.getLength())
.isdir(fs.getFileType().equals(FileType.IS_DIR))
.replication(fs.getBlockReplication())
.blocksize(fs.getBlocksize())
.mtime(fs.getModificationTime())
.atime(fs.getAccessTime())
.perm(convert(fs.getPermission()))
.flags(flags)
.owner(fs.getOwner())
.group(fs.getGroup())
.symlink(FileType.IS_SYMLINK.equals(fs.getFileType())
? fs.getSymlink().toByteArray()
: null)
.path(fs.getPath().toByteArray())
.fileId(fs.hasFileId()
? fs.getFileId()
: HdfsConstants.GRANDFATHER_INODE_ID)
.locations(fs.hasLocations() ? convert(fs.getLocations()) : null)
.children(fs.hasChildrenNum() ? fs.getChildrenNum() : -1)
.feInfo(fs.hasFileEncryptionInfo()
? convert(fs.getFileEncryptionInfo())
: null)
.storagePolicy(fs.hasStoragePolicy()
? (byte) fs.getStoragePolicy()
: HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED)
.ecPolicy(fs.hasEcPolicy()
? convertErasureCodingPolicy(fs.getEcPolicy())
: null)
.build();
if (fs.hasNamespace()) {
hdfsFileStatus.setNamespace(fs.getNamespace());
}
return hdfsFileStatus;
}
private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) {
EnumSet<HdfsFileStatus.Flags> f =
EnumSet.noneOf(HdfsFileStatus.Flags.class);
for (HdfsFileStatusProto.Flags pbf : HdfsFileStatusProto.Flags.values()) {
if ((pbf.getNumber() & flags) != 0) {
switch (pbf) {
case HAS_ACL:
f.add(HdfsFileStatus.Flags.HAS_ACL);
break;
case HAS_CRYPT:
f.add(HdfsFileStatus.Flags.HAS_CRYPT);
break;
case HAS_EC:
f.add(HdfsFileStatus.Flags.HAS_EC);
break;
case SNAPSHOT_ENABLED:
f.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
break;
default:
// ignore unknown
break;
}
}
}
return f;
}
private static EnumSet<HdfsFileStatus.Flags> convertFlags(
FsPermissionProto pbp) {
EnumSet<HdfsFileStatus.Flags> f =
EnumSet.noneOf(HdfsFileStatus.Flags.class);
FsPermission p = new FsPermissionExtension((short)pbp.getPerm());
if (p.getAclBit()) {
f.add(HdfsFileStatus.Flags.HAS_ACL);
}
if (p.getEncryptedBit()) {
f.add(HdfsFileStatus.Flags.HAS_CRYPT);
}
if (p.getErasureCodedBit()) {
f.add(HdfsFileStatus.Flags.HAS_EC);
}
return f;
}
public static CorruptFileBlocks convert(CorruptFileBlocksProto c) {
if (c == null)
return null;
List<String> fileList = c.getFilesList();
return new CorruptFileBlocks(fileList.toArray(new String[fileList.size()]),
c.getCookie());
}
public static ContentSummary convert(ContentSummaryProto cs) {
if (cs == null) return null;
ContentSummary.Builder builder = new ContentSummary.Builder();
builder.length(cs.getLength()).
fileCount(cs.getFileCount()).
directoryCount(cs.getDirectoryCount()).
snapshotLength(cs.getSnapshotLength()).
snapshotFileCount(cs.getSnapshotFileCount()).
snapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
quota(cs.getQuota()).
spaceConsumed(cs.getSpaceConsumed()).
spaceQuota(cs.getSpaceQuota()).
erasureCodingPolicy(cs.getErasureCodingPolicy());
if (cs.hasTypeQuotaInfos()) {
addStorageTypes(cs.getTypeQuotaInfos(), builder);
}
return builder.build();
}
public static QuotaUsage convert(QuotaUsageProto qu) {
if (qu == null) {
return null;
}
QuotaUsage.Builder builder = new QuotaUsage.Builder();
builder.fileAndDirectoryCount(qu.getFileAndDirectoryCount()).
quota(qu.getQuota()).
spaceConsumed(qu.getSpaceConsumed()).
spaceQuota(qu.getSpaceQuota());
if (qu.hasTypeQuotaInfos()) {
addStorageTypes(qu.getTypeQuotaInfos(), builder);
}
return builder.build();
}
public static QuotaUsageProto convert(QuotaUsage qu) {
if (qu == null) {
return null;
}
QuotaUsageProto.Builder builder = QuotaUsageProto.newBuilder();
builder.setFileAndDirectoryCount(qu.getFileAndDirectoryCount()).
setQuota(qu.getQuota()).
setSpaceConsumed(qu.getSpaceConsumed()).
setSpaceQuota(qu.getSpaceQuota());
if (qu.isTypeQuotaSet() || qu.isTypeConsumedAvailable()) {
builder.setTypeQuotaInfos(getBuilder(qu));
}
return builder.build();
}
public static ReencryptActionProto convert(ReencryptAction a) {
switch (a) {
case CANCEL:
return ReencryptActionProto.CANCEL_REENCRYPT;
case START:
return ReencryptActionProto.START_REENCRYPT;
default:
throw new IllegalArgumentException("Unexpected value: " + a);
}
}
public static RollingUpgradeActionProto convert(RollingUpgradeAction a) {
switch (a) {
case QUERY:
return RollingUpgradeActionProto.QUERY;
case PREPARE:
return RollingUpgradeActionProto.START;
case FINALIZE:
return RollingUpgradeActionProto.FINALIZE;
default:
throw new IllegalArgumentException("Unexpected value: " + a);
}
}
public static RollingUpgradeInfo convert(RollingUpgradeInfoProto proto) {
RollingUpgradeStatusProto status = proto.getStatus();
return new RollingUpgradeInfo(status.getBlockPoolId(),
proto.getCreatedRollbackImages(),
proto.getStartTime(), proto.getFinalizeTime());
}
public static DatanodeStorageReport[] convertDatanodeStorageReports(
List<DatanodeStorageReportProto> protos) {
final DatanodeStorageReport[] reports
= new DatanodeStorageReport[protos.size()];
for(int i = 0; i < reports.length; i++) {
reports[i] = convertDatanodeStorageReport(protos.get(i));
}
return reports;
}
public static DatanodeStorageReport convertDatanodeStorageReport(
DatanodeStorageReportProto proto) {
return new DatanodeStorageReport(
convert(proto.getDatanodeInfo()),
convertStorageReports(proto.getStorageReportsList()));
}
public static StorageReport[] convertStorageReports(
List<StorageReportProto> list) {
final StorageReport[] report = new StorageReport[list.size()];
for (int i = 0; i < report.length; i++) {
report[i] = convert(list.get(i));
}
return report;
}
public static StorageReport convert(StorageReportProto p) {
long nonDfsUsed = p.hasNonDfsUsed() ? p.getNonDfsUsed() : p.getCapacity()
- p.getDfsUsed() - p.getRemaining();
return new StorageReport(p.hasStorage() ? convert(p.getStorage())
: new DatanodeStorage(p.getStorageUuid()), p.getFailed(),
p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
p.getBlockPoolUsed(), nonDfsUsed, p.getMount());
}
public static DatanodeStorage convert(DatanodeStorageProto s) {
return new DatanodeStorage(s.getStorageUuid(),
convertState(s.getState()), convertStorageType(s.getStorageType()));
}
private static State convertState(StorageState state) {
switch(state) {
case READ_ONLY_SHARED:
return State.READ_ONLY_SHARED;
case NORMAL:
default:
return State.NORMAL;
}
}
public static SafeModeActionProto convert(SafeModeAction a) {
switch (a) {
case SAFEMODE_LEAVE:
return SafeModeActionProto.SAFEMODE_LEAVE;
case SAFEMODE_ENTER:
return SafeModeActionProto.SAFEMODE_ENTER;
case SAFEMODE_GET:
return SafeModeActionProto.SAFEMODE_GET;
case SAFEMODE_FORCE_EXIT:
return SafeModeActionProto.SAFEMODE_FORCE_EXIT;
default:
throw new IllegalArgumentException("Unexpected SafeModeAction :" + a);
}
}
public static DatanodeInfo[] convert(List<DatanodeInfoProto> list) {
DatanodeInfo[] info = new DatanodeInfo[list.size()];
for (int i = 0; i < info.length; i++) {
info[i] = convert(list.get(i));
}
return info;
}
public static long[] convert(GetFsStatsResponseProto res) {
long[] result = new long[ClientProtocol.STATS_ARRAY_LENGTH];
result[ClientProtocol.GET_STATS_CAPACITY_IDX] = res.getCapacity();
result[ClientProtocol.GET_STATS_USED_IDX] = res.getUsed();
result[ClientProtocol.GET_STATS_REMAINING_IDX] = res.getRemaining();
result[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX] =
res.getUnderReplicated();
result[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX] =
res.getCorruptBlocks();
result[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX] =
res.getMissingBlocks();
result[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX] =
res.getMissingReplOneBlocks();
result[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX] =
res.hasBlocksInFuture() ? res.getBlocksInFuture() : 0;
result[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX] =
res.getPendingDeletionBlocks();
return result;
}
public static ReplicatedBlockStats convert(
GetFsReplicatedBlockStatsResponseProto res) {
if (res.hasHighestPrioLowRedundancyBlocks()) {
return new ReplicatedBlockStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
res.getPendingDeletionBlocks(),
res.getHighestPrioLowRedundancyBlocks());
}
return new ReplicatedBlockStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
res.getPendingDeletionBlocks());
}
public static ECBlockGroupStats convert(
GetFsECBlockGroupStatsResponseProto res) {
if (res.hasHighestPrioLowRedundancyBlocks()) {
return new ECBlockGroupStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
res.getHighestPrioLowRedundancyBlocks());
}
return new ECBlockGroupStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getBlocksInFuture(), res.getPendingDeletionBlocks());
}
public static DatanodeReportTypeProto convert(DatanodeReportType t) {
switch (t) {
case ALL: return DatanodeReportTypeProto.ALL;
case LIVE: return DatanodeReportTypeProto.LIVE;
case DEAD: return DatanodeReportTypeProto.DEAD;
case DECOMMISSIONING: return DatanodeReportTypeProto.DECOMMISSIONING;
case ENTERING_MAINTENANCE:
return DatanodeReportTypeProto.ENTERING_MAINTENANCE;
case IN_MAINTENANCE:
return DatanodeReportTypeProto.IN_MAINTENANCE;
default:
throw new IllegalArgumentException("Unexpected data type report:" + t);
}
}
public static DirectoryListing convert(DirectoryListingProto dl) {
if (dl == null)
return null;
List<HdfsFileStatusProto> partList = dl.getPartialListingList();
return new DirectoryListing(partList.isEmpty()
? new HdfsFileStatus[0]
: convert(partList.toArray(new HdfsFileStatusProto[partList.size()])),
dl.getRemainingEntries());
}
public static HdfsFileStatus[] convert(HdfsFileStatusProto[] fs) {
if (fs == null) return null;
final int len = fs.length;
HdfsFileStatus[] result = new HdfsFileStatus[len];
for (int i = 0; i < len; ++i) {
result[i] = convert(fs[i]);
}
return result;
}
public static List<HdfsFileStatus> convertHdfsFileStatus(
List<HdfsFileStatusProto> fs) {
if (fs == null) {
return null;
}
List<HdfsFileStatus> result = Lists.newArrayListWithCapacity(fs.size());
for (HdfsFileStatusProto proto : fs) {
result.add(convert(proto));
}
return result;
}
// The creatFlag field in PB is a bitmask whose values are the same a the
// emum values of CreateFlag
public static int convertCreateFlag(EnumSetWritable<CreateFlag> flag) {
int value = 0;
if (flag.contains(CreateFlag.APPEND)) {
value |= CreateFlagProto.APPEND.getNumber();
}
if (flag.contains(CreateFlag.CREATE)) {
value |= CreateFlagProto.CREATE.getNumber();
}
if (flag.contains(CreateFlag.OVERWRITE)) {
value |= CreateFlagProto.OVERWRITE.getNumber();
}
if (flag.contains(CreateFlag.LAZY_PERSIST)) {
value |= CreateFlagProto.LAZY_PERSIST.getNumber();
}
if (flag.contains(CreateFlag.NEW_BLOCK)) {
value |= CreateFlagProto.NEW_BLOCK.getNumber();
}
if (flag.contains(CreateFlag.SHOULD_REPLICATE)) {
value |= CreateFlagProto.SHOULD_REPLICATE.getNumber();
}
return value;
}
public static FsServerDefaults convert(FsServerDefaultsProto fs) {
if (fs == null) return null;
return new FsServerDefaults(
fs.getBlockSize(), fs.getBytesPerChecksum(),
fs.getWritePacketSize(), (short) fs.getReplication(),
fs.getFileBufferSize(),
fs.getEncryptDataTransfer(),
fs.getTrashInterval(),
convert(fs.getChecksumType()),
fs.hasKeyProviderUri() ? fs.getKeyProviderUri() : null,
(byte) fs.getPolicyId(),
fs.getSnapshotTrashRootEnabled());
}
public static List<CryptoProtocolVersionProto> convert(
CryptoProtocolVersion[] versions) {
List<CryptoProtocolVersionProto> protos =
Lists.newArrayListWithCapacity(versions.length);
for (CryptoProtocolVersion v: versions) {
protos.add(convert(v));
}
return protos;
}
static List<StorageTypesProto> convert(StorageType[][] types) {
List<StorageTypesProto> list = Lists.newArrayList();
if (types != null) {
for (StorageType[] ts : types) {
StorageTypesProto.Builder builder = StorageTypesProto.newBuilder();
builder.addAllStorageTypes(convertStorageTypes(ts));
list.add(builder.build());
}
}
return list;
}
public static BlockStoragePolicyProto convert(BlockStoragePolicy policy) {
BlockStoragePolicyProto.Builder builder = BlockStoragePolicyProto
.newBuilder().setPolicyId(policy.getId()).setName(policy.getName());
// creation storage types
StorageTypesProto creationProto = convert(policy.getStorageTypes());
Preconditions.checkArgument(creationProto != null);
builder.setCreationPolicy(creationProto);
// creation fallback
StorageTypesProto creationFallbackProto = convert(
policy.getCreationFallbacks());
if (creationFallbackProto != null) {
builder.setCreationFallbackPolicy(creationFallbackProto);
}
// replication fallback
StorageTypesProto replicationFallbackProto = convert(
policy.getReplicationFallbacks());
if (replicationFallbackProto != null) {
builder.setReplicationFallbackPolicy(replicationFallbackProto);
}
return builder.build();
}
public static StorageTypesProto convert(StorageType[] types) {
if (types == null || types.length == 0) {
return null;
}
List<StorageTypeProto> list = convertStorageTypes(types);
return StorageTypesProto.newBuilder().addAllStorageTypes(list).build();
}
public static DatanodeID[] convert(DatanodeIDProto[] did) {
if (did == null) return null;
final int len = did.length;
DatanodeID[] result = new DatanodeID[len];
for (int i = 0; i < len; ++i) {
result[i] = convert(did[i]);
}
return result;
}
// Block
public static BlockProto convert(Block b) {
return BlockProto.newBuilder().setBlockId(b.getBlockId())
.setGenStamp(b.getGenerationStamp()).setNumBytes(b.getNumBytes())
.build();
}
public static Block convert(BlockProto b) {
return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp());
}
public static BlockTypeProto convert(BlockType blockType) {
switch (blockType) {
case CONTIGUOUS: return BlockTypeProto.CONTIGUOUS;
case STRIPED: return BlockTypeProto.STRIPED;
default:
throw new IllegalArgumentException("Unexpected block type: " + blockType);
}
}
public static BlockType convert(BlockTypeProto blockType) {
switch (blockType.getNumber()) {
case BlockTypeProto.CONTIGUOUS_VALUE: return BlockType.CONTIGUOUS;
case BlockTypeProto.STRIPED_VALUE: return BlockType.STRIPED;
default:
throw new IllegalArgumentException("Unexpected block type: " + blockType);
}
}
static public DatanodeInfo[] convert(DatanodeInfoProto di[]) {
if (di == null) return null;
DatanodeInfo[] result = new DatanodeInfo[di.length];
for (int i = 0; i < di.length; i++) {
result[i] = convert(di[i]);
}
return result;
}
public static DatanodeStorageReportProto convertDatanodeStorageReport(
DatanodeStorageReport report) {
return DatanodeStorageReportProto.newBuilder()
.setDatanodeInfo(convert(report.getDatanodeInfo()))
.addAllStorageReports(convertStorageReports(report.getStorageReports()))
.build();
}
public static List<DatanodeStorageReportProto> convertDatanodeStorageReports(
DatanodeStorageReport[] reports) {
final List<DatanodeStorageReportProto> protos
= new ArrayList<>(reports.length);
for (DatanodeStorageReport report : reports) {
protos.add(convertDatanodeStorageReport(report));
}
return protos;
}
public static LocatedBlock[] convertLocatedBlock(LocatedBlockProto[] lb) {
if (lb == null) return null;
return convertLocatedBlock(Arrays.asList(lb)).toArray(
new LocatedBlock[lb.length]);
}
public static LocatedBlocksProto convert(LocatedBlocks lb) {
if (lb == null) {
return null;
}
LocatedBlocksProto.Builder builder =
LocatedBlocksProto.newBuilder();
if (lb.getLastLocatedBlock() != null) {
builder.setLastBlock(
convertLocatedBlock(lb.getLastLocatedBlock()));
}
if (lb.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo()));
}
if (lb.getErasureCodingPolicy() != null) {
builder.setEcPolicy(convertErasureCodingPolicy(
lb.getErasureCodingPolicy()));
}
return builder.setFileLength(lb.getFileLength())
.setUnderConstruction(lb.isUnderConstruction())
.addAllBlocks(convertLocatedBlocks2(lb.getLocatedBlocks()))
.setIsLastBlockComplete(lb.isLastBlockComplete()).build();
}
public static DataEncryptionKeyProto convert(DataEncryptionKey bet) {
DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder()
.setKeyId(bet.keyId)
.setBlockPoolId(bet.blockPoolId)
.setNonce(getByteString(bet.nonce))
.setEncryptionKey(getByteString(bet.encryptionKey))
.setExpiryDate(bet.expiryDate);
if (bet.encryptionAlgorithm != null) {
b.setEncryptionAlgorithm(bet.encryptionAlgorithm);
}
return b.build();
}
public static FsServerDefaultsProto convert(FsServerDefaults fs) {
if (fs == null) return null;
FsServerDefaultsProto.Builder builder = FsServerDefaultsProto.newBuilder().
setBlockSize(fs.getBlockSize()).
setBytesPerChecksum(fs.getBytesPerChecksum()).
setWritePacketSize(fs.getWritePacketSize())
.setReplication(fs.getReplication())
.setFileBufferSize(fs.getFileBufferSize())
.setEncryptDataTransfer(fs.getEncryptDataTransfer())
.setTrashInterval(fs.getTrashInterval())
.setChecksumType(convert(fs.getChecksumType()))
.setPolicyId(fs.getDefaultStoragePolicyId())
.setSnapshotTrashRootEnabled(fs.getSnapshotTrashRootEnabled());
if (fs.getKeyProviderUri() != null) {
builder.setKeyProviderUri(fs.getKeyProviderUri());
}
return builder.build();
}
public static EnumSetWritable<CreateFlag> convertCreateFlag(int flag) {
EnumSet<CreateFlag> result =
EnumSet.noneOf(CreateFlag.class);
if ((flag & CreateFlagProto.APPEND_VALUE) == CreateFlagProto.APPEND_VALUE) {
result.add(CreateFlag.APPEND);
}
if ((flag & CreateFlagProto.CREATE_VALUE) == CreateFlagProto.CREATE_VALUE) {
result.add(CreateFlag.CREATE);
}
if ((flag & CreateFlagProto.OVERWRITE_VALUE)
== CreateFlagProto.OVERWRITE_VALUE) {
result.add(CreateFlag.OVERWRITE);
}
if ((flag & CreateFlagProto.LAZY_PERSIST_VALUE)
== CreateFlagProto.LAZY_PERSIST_VALUE) {
result.add(CreateFlag.LAZY_PERSIST);
}
if ((flag & CreateFlagProto.NEW_BLOCK_VALUE)
== CreateFlagProto.NEW_BLOCK_VALUE) {
result.add(CreateFlag.NEW_BLOCK);
}
if ((flag & CreateFlagProto.SHOULD_REPLICATE.getNumber())
== CreateFlagProto.SHOULD_REPLICATE.getNumber()) {
result.add(CreateFlag.SHOULD_REPLICATE);
}
return new EnumSetWritable<>(result, CreateFlag.class);
}
public static EnumSet<CacheFlag> convertCacheFlags(int flags) {
EnumSet<CacheFlag> result = EnumSet.noneOf(CacheFlag.class);
if ((flags & CacheFlagProto.FORCE_VALUE) == CacheFlagProto.FORCE_VALUE) {
result.add(CacheFlag.FORCE);
}
return result;
}
public static HdfsFileStatusProto convert(HdfsFileStatus fs) {
if (fs == null)
return null;
FileType fType = FileType.IS_FILE;
if (fs.isDirectory()) {
fType = FileType.IS_DIR;
} else if (fs.isSymlink()) {
fType = FileType.IS_SYMLINK;
}
HdfsFileStatusProto.Builder builder =
HdfsFileStatusProto.newBuilder().
setLength(fs.getLen()).
setFileType(fType).
setBlockReplication(fs.getReplication()).
setBlocksize(fs.getBlockSize()).
setModificationTime(fs.getModificationTime()).
setAccessTime(fs.getAccessTime()).
setPermission(convert(fs.getPermission())).
setOwnerBytes(getFixedByteString(fs.getOwner())).
setGroupBytes(getFixedByteString(fs.getGroup())).
setFileId(fs.getFileId()).
setChildrenNum(fs.getChildrenNum()).
setPath(getByteString(fs.getLocalNameInBytes())).
setStoragePolicy(fs.getStoragePolicy());
if (fs.isSymlink()) {
builder.setSymlink(getByteString(fs.getSymlinkInBytes()));
}
if (fs.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
}
if (fs instanceof HdfsLocatedFileStatus) {
final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs;
LocatedBlocks locations = lfs.getLocatedBlocks();
if (locations != null) {
builder.setLocations(convert(locations));
}
}
if(fs.getErasureCodingPolicy() != null) {
builder.setEcPolicy(convertErasureCodingPolicy(
fs.getErasureCodingPolicy()));
}
int flags = fs.hasAcl() ? HdfsFileStatusProto.Flags.HAS_ACL_VALUE : 0;
flags |= fs.isEncrypted() ? HdfsFileStatusProto.Flags.HAS_CRYPT_VALUE : 0;
flags |= fs.isErasureCoded() ? HdfsFileStatusProto.Flags.HAS_EC_VALUE : 0;
flags |= fs.isSnapshotEnabled() ? HdfsFileStatusProto.Flags
.SNAPSHOT_ENABLED_VALUE : 0;
builder.setFlags(flags);
if (fs.getNamespace() != null && !fs.getNamespace().isEmpty()) {
builder.setNamespace(fs.getNamespace());
}
return builder.build();
}
public static SnapshottableDirectoryStatusProto convert(
SnapshottableDirectoryStatus status) {
if (status == null) {
return null;
}
int snapshotNumber = status.getSnapshotNumber();
int snapshotQuota = status.getSnapshotQuota();
byte[] parentFullPath = status.getParentFullPath();
ByteString parentFullPathBytes = getByteString(
parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
HdfsFileStatusProto fs = convert(status.getDirStatus());
SnapshottableDirectoryStatusProto.Builder builder =
SnapshottableDirectoryStatusProto
.newBuilder()
.setSnapshotNumber(snapshotNumber)
.setSnapshotQuota(snapshotQuota)
.setParentFullpath(parentFullPathBytes)
.setDirStatus(fs);
return builder.build();
}
public static HdfsProtos.SnapshotStatusProto convert(SnapshotStatus status) {
if (status == null) {
return null;
}
byte[] parentFullPath = status.getParentFullPath();
ByteString parentFullPathBytes = getByteString(
parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
HdfsFileStatusProto fs = convert(status.getDirStatus());
HdfsProtos.SnapshotStatusProto.Builder builder =
HdfsProtos.SnapshotStatusProto
.newBuilder()
.setSnapshotID(status.getSnapshotID())
.setParentFullpath(parentFullPathBytes)
.setIsDeleted(status.isDeleted())
.setDirStatus(fs);
return builder.build();
}
public static HdfsFileStatusProto[] convert(HdfsFileStatus[] fs) {
if (fs == null) return null;
final int len = fs.length;
HdfsFileStatusProto[] result = new HdfsFileStatusProto[len];
for (int i = 0; i < len; ++i) {
result[i] = convert(fs[i]);
}
return result;
}
public static DirectoryListingProto convert(DirectoryListing d) {
if (d == null)
return null;
return DirectoryListingProto.newBuilder().
addAllPartialListing(Arrays.asList(
convert(d.getPartialListing()))).
setRemainingEntries(d.getRemainingEntries()).
build();
}
public static GetFsStatsResponseProto convert(long[] fsStats) {
GetFsStatsResponseProto.Builder result = GetFsStatsResponseProto
.newBuilder();
if (fsStats.length >= ClientProtocol.GET_STATS_CAPACITY_IDX + 1)
result.setCapacity(fsStats[ClientProtocol.GET_STATS_CAPACITY_IDX]);
if (fsStats.length >= ClientProtocol.GET_STATS_USED_IDX + 1)
result.setUsed(fsStats[ClientProtocol.GET_STATS_USED_IDX]);
if (fsStats.length >= ClientProtocol.GET_STATS_REMAINING_IDX + 1)
result.setRemaining(fsStats[ClientProtocol.GET_STATS_REMAINING_IDX]);
if (fsStats.length >= ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX + 1)
result.setUnderReplicated(
fsStats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]);
if (fsStats.length >= ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX + 1)
result.setCorruptBlocks(
fsStats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX]);
if (fsStats.length >= ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX + 1)
result.setMissingBlocks(
fsStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX]);
if (fsStats.length >=
ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX + 1)
result.setMissingReplOneBlocks(
fsStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]);
if (fsStats.length >=
ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX + 1) {
result.setBlocksInFuture(
fsStats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX]);
}
if (fsStats.length >=
ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX + 1) {
result.setPendingDeletionBlocks(
fsStats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX]);
}
return result.build();
}
public static GetFsReplicatedBlockStatsResponseProto convert(
ReplicatedBlockStats replicatedBlockStats) {
GetFsReplicatedBlockStatsResponseProto.Builder result =
GetFsReplicatedBlockStatsResponseProto.newBuilder();
result.setLowRedundancy(
replicatedBlockStats.getLowRedundancyBlocks());
result.setCorruptBlocks(
replicatedBlockStats.getCorruptBlocks());
result.setMissingBlocks(
replicatedBlockStats.getMissingReplicaBlocks());
result.setMissingReplOneBlocks(
replicatedBlockStats.getMissingReplicationOneBlocks());
result.setBlocksInFuture(
replicatedBlockStats.getBytesInFutureBlocks());
result.setPendingDeletionBlocks(
replicatedBlockStats.getPendingDeletionBlocks());
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
result.setHighestPrioLowRedundancyBlocks(
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
}
return result.build();
}
public static GetFsECBlockGroupStatsResponseProto convert(
ECBlockGroupStats ecBlockGroupStats) {
GetFsECBlockGroupStatsResponseProto.Builder result =
GetFsECBlockGroupStatsResponseProto.newBuilder();
result.setLowRedundancy(
ecBlockGroupStats.getLowRedundancyBlockGroups());
result.setCorruptBlocks(ecBlockGroupStats.getCorruptBlockGroups());
result.setMissingBlocks(ecBlockGroupStats.getMissingBlockGroups());
result.setBlocksInFuture(
ecBlockGroupStats.getBytesInFutureBlockGroups());
result.setPendingDeletionBlocks(
ecBlockGroupStats.getPendingDeletionBlocks());
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
result.setHighestPrioLowRedundancyBlocks(
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());
}
return result.build();
}
public static DatanodeReportType convert(DatanodeReportTypeProto t) {
switch (t) {
case ALL: return DatanodeReportType.ALL;
case LIVE: return DatanodeReportType.LIVE;
case DEAD: return DatanodeReportType.DEAD;
case DECOMMISSIONING: return DatanodeReportType.DECOMMISSIONING;
case ENTERING_MAINTENANCE:
return DatanodeReportType.ENTERING_MAINTENANCE;
case IN_MAINTENANCE:
return DatanodeReportType.IN_MAINTENANCE;
default:
throw new IllegalArgumentException("Unexpected data type report:" + t);
}
}
public static SafeModeAction convert(
SafeModeActionProto a) {
switch (a) {
case SAFEMODE_LEAVE:
return SafeModeAction.SAFEMODE_LEAVE;
case SAFEMODE_ENTER:
return SafeModeAction.SAFEMODE_ENTER;
case SAFEMODE_GET:
return SafeModeAction.SAFEMODE_GET;
case SAFEMODE_FORCE_EXIT:
return SafeModeAction.SAFEMODE_FORCE_EXIT;
default:
throw new IllegalArgumentException("Unexpected SafeModeAction :" + a);
}
}
public static ReencryptAction convert(ReencryptActionProto a) {
switch (a) {
case CANCEL_REENCRYPT:
return ReencryptAction.CANCEL;
case START_REENCRYPT:
return ReencryptAction.START;
default:
throw new IllegalArgumentException("Unexpected value: " + a);
}
}
public static RollingUpgradeAction convert(RollingUpgradeActionProto a) {
switch (a) {
case QUERY:
return RollingUpgradeAction.QUERY;
case START:
return RollingUpgradeAction.PREPARE;
case FINALIZE:
return RollingUpgradeAction.FINALIZE;
default:
throw new IllegalArgumentException("Unexpected value: " + a);
}
}
public static RollingUpgradeStatusProto convertRollingUpgradeStatus(
RollingUpgradeStatus status) {
return RollingUpgradeStatusProto.newBuilder()
.setBlockPoolId(status.getBlockPoolId())
.setFinalized(status.isFinalized())
.build();
}
public static RollingUpgradeStatus convert(RollingUpgradeStatusProto proto) {
return new RollingUpgradeStatus(proto.getBlockPoolId(),
proto.getFinalized());
}
public static RollingUpgradeInfoProto convert(RollingUpgradeInfo info) {
return RollingUpgradeInfoProto.newBuilder()
.setStatus(convertRollingUpgradeStatus(info))
.setCreatedRollbackImages(info.createdRollbackImages())
.setStartTime(info.getStartTime())
.setFinalizeTime(info.getFinalizeTime())
.build();
}
public static CorruptFileBlocksProto convert(CorruptFileBlocks c) {
if (c == null)
return null;
return CorruptFileBlocksProto.newBuilder().
addAllFiles(Arrays.asList(c.getFiles())).
setCookie(c.getCookie()).
build();
}
public static ContentSummaryProto convert(ContentSummary cs) {
if (cs == null) return null;
ContentSummaryProto.Builder builder = ContentSummaryProto.newBuilder();
builder.setLength(cs.getLength()).
setFileCount(cs.getFileCount()).
setDirectoryCount(cs.getDirectoryCount()).
setSnapshotLength(cs.getSnapshotLength()).
setSnapshotFileCount(cs.getSnapshotFileCount()).
setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
setQuota(cs.getQuota()).
setSpaceConsumed(cs.getSpaceConsumed()).
setSpaceQuota(cs.getSpaceQuota()).
setErasureCodingPolicy(cs.getErasureCodingPolicy());
if (cs.isTypeQuotaSet() || cs.isTypeConsumedAvailable()) {
builder.setTypeQuotaInfos(getBuilder(cs));
}
return builder.build();
}
private static void addStorageTypes(
HdfsProtos.StorageTypeQuotaInfosProto typeQuotaInfos,
QuotaUsage.Builder builder) {
for (HdfsProtos.StorageTypeQuotaInfoProto info :
typeQuotaInfos.getTypeQuotaInfoList()) {
StorageType type = convertStorageType(info.getType());
builder.typeConsumed(type, info.getConsumed());
builder.typeQuota(type, info.getQuota());
}
}
private static HdfsProtos.StorageTypeQuotaInfosProto.Builder getBuilder(
QuotaUsage qu) {
HdfsProtos.StorageTypeQuotaInfosProto.Builder isb =
HdfsProtos.StorageTypeQuotaInfosProto.newBuilder();
for (StorageType t: StorageType.getTypesSupportingQuota()) {
HdfsProtos.StorageTypeQuotaInfoProto info =
HdfsProtos.StorageTypeQuotaInfoProto.newBuilder().
setType(convertStorageType(t)).
setConsumed(qu.getTypeConsumed(t)).
setQuota(qu.getTypeQuota(t)).
build();
isb.addTypeQuotaInfo(info);
}
return isb;
}
public static DatanodeStorageProto convert(DatanodeStorage s) {
return DatanodeStorageProto.newBuilder()
.setState(convertState(s.getState()))
.setStorageType(convertStorageType(s.getStorageType()))
.setStorageUuid(s.getStorageID()).build();
}
private static StorageState convertState(State state) {
switch(state) {
case READ_ONLY_SHARED:
return StorageState.READ_ONLY_SHARED;
case NORMAL:
default:
return StorageState.NORMAL;
}
}
public static StorageReportProto convert(StorageReport r) {
StorageReportProto.Builder builder = StorageReportProto.newBuilder()
.setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
.setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
.setStorageUuid(r.getStorage().getStorageID())
.setStorage(convert(r.getStorage()))
.setNonDfsUsed(r.getNonDfsUsed())
.setMount(r.getMount());
return builder.build();
}
public static List<StorageReportProto> convertStorageReports(
StorageReport[] storages) {
final List<StorageReportProto> protos = new ArrayList<>(storages.length);
for (StorageReport storage : storages) {
protos.add(convert(storage));
}
return protos;
}
public static SnapshottableDirectoryListingProto convert(
SnapshottableDirectoryStatus[] status) {
if (status == null)
return null;
SnapshottableDirectoryStatusProto[] protos =
new SnapshottableDirectoryStatusProto[status.length];
for (int i = 0; i < status.length; i++) {
protos[i] = convert(status[i]);
}
List<SnapshottableDirectoryStatusProto> protoList = Arrays.asList(protos);
return SnapshottableDirectoryListingProto.newBuilder()
.addAllSnapshottableDirListing(protoList).build();
}
public static HdfsProtos.SnapshotListingProto convert(
SnapshotStatus[] status) {
if (status == null) {
return null;
}
HdfsProtos.SnapshotStatusProto[] protos =
new HdfsProtos.SnapshotStatusProto[status.length];
for (int i = 0; i < status.length; i++) {
protos[i] = convert(status[i]);
}
List<SnapshotStatusProto> protoList = Arrays.asList(protos);
return SnapshotListingProto.newBuilder()
.addAllSnapshotListing(protoList).build();
}
public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) {
if (entry == null) {
return null;
}
ByteString sourcePath = getByteString(entry.getSourcePath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getSourcePath());
String modification = entry.getType().getLabel();
SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
.newBuilder().setFullpath(sourcePath)
.setModificationLabel(modification);
if (entry.getType() == DiffType.RENAME) {
ByteString targetPath =
getByteString(entry.getTargetPath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getTargetPath());
builder.setTargetPath(targetPath);
}
return builder.build();
}
public static SnapshotDiffReportListingEntryProto convert(
DiffReportListingEntry entry) {
if (entry == null) {
return null;
}
ByteString sourcePath = getByteString(
entry.getSourcePath() == null ? DFSUtilClient.EMPTY_BYTES :
DFSUtilClient.byteArray2bytes(entry.getSourcePath()));
long dirId = entry.getDirId();
long fileId = entry.getFileId();
boolean isReference = entry.isReference();
ByteString targetPath = getByteString(
entry.getTargetPath() == null ? DFSUtilClient.EMPTY_BYTES :
DFSUtilClient.byteArray2bytes(entry.getTargetPath()));
SnapshotDiffReportListingEntryProto.Builder builder =
SnapshotDiffReportListingEntryProto.newBuilder().setFullpath(sourcePath)
.setDirId(dirId).setFileId(fileId).setIsReference(isReference)
.setTargetPath(targetPath);
return builder.build();
}
public static SnapshotDiffReportListingProto convert(
SnapshotDiffReportListing report) {
if (report == null) {
return null;
}
ByteString startPath = getByteString(
report.getLastPath() == null ? DFSUtilClient.EMPTY_BYTES :
report.getLastPath());
List<DiffReportListingEntry> modifiedEntries = report.getModifyList();
List<DiffReportListingEntry> createdEntries = report.getCreateList();
List<DiffReportListingEntry> deletedEntries = report.getDeleteList();
List<SnapshotDiffReportListingEntryProto> modifiedEntryProtos =
new ChunkedArrayList<>();
for (DiffReportListingEntry entry : modifiedEntries) {
SnapshotDiffReportListingEntryProto entryProto = convert(entry);
if (entryProto != null) {
modifiedEntryProtos.add(entryProto);
}
}
List<SnapshotDiffReportListingEntryProto> createdEntryProtos =
new ChunkedArrayList<>();
for (DiffReportListingEntry entry : createdEntries) {
SnapshotDiffReportListingEntryProto entryProto = convert(entry);
if (entryProto != null) {
createdEntryProtos.add(entryProto);
}
}
List<SnapshotDiffReportListingEntryProto> deletedEntryProtos =
new ChunkedArrayList<>();
for (DiffReportListingEntry entry : deletedEntries) {
SnapshotDiffReportListingEntryProto entryProto = convert(entry);
if (entryProto != null) {
deletedEntryProtos.add(entryProto);
}
}
return SnapshotDiffReportListingProto.newBuilder()
.addAllModifiedEntries(modifiedEntryProtos)
.addAllCreatedEntries(createdEntryProtos)
.addAllDeletedEntries(deletedEntryProtos)
.setIsFromEarlier(report.getIsFromEarlier())
.setCursor(HdfsProtos.SnapshotDiffReportCursorProto.newBuilder()
.setStartPath(startPath)
.setIndex(report.getLastIndex()).build())
.build();
}
public static SnapshotDiffReportProto convert(SnapshotDiffReport report) {
if (report == null) {
return null;
}
List<DiffReportEntry> entries = report.getDiffList();
List<SnapshotDiffReportEntryProto> entryProtos = new ChunkedArrayList<>();
for (DiffReportEntry entry : entries) {
SnapshotDiffReportEntryProto entryProto = convert(entry);
if (entryProto != null)
entryProtos.add(entryProto);
}
return SnapshotDiffReportProto.newBuilder()
.setSnapshotRoot(report.getSnapshotRoot())
.setFromSnapshot(report.getFromSnapshot())
.setToSnapshot(report.getLaterSnapshotName())
.addAllDiffReportEntries(entryProtos).build();
}
public static CacheDirectiveStatsProto convert(CacheDirectiveStats stats) {
CacheDirectiveStatsProto.Builder builder =
CacheDirectiveStatsProto.newBuilder();
builder.setBytesNeeded(stats.getBytesNeeded());
builder.setBytesCached(stats.getBytesCached());
builder.setFilesNeeded(stats.getFilesNeeded());
builder.setFilesCached(stats.getFilesCached());
builder.setHasExpired(stats.hasExpired());
return builder.build();
}
public static CacheDirectiveEntryProto convert(CacheDirectiveEntry entry) {
CacheDirectiveEntryProto.Builder builder =
CacheDirectiveEntryProto.newBuilder();
builder.setInfo(convert(entry.getInfo()));
builder.setStats(convert(entry.getStats()));
return builder.build();
}
public static boolean[] convertBooleanList(
List<Boolean> targetPinningsList) {
final boolean[] targetPinnings = new boolean[targetPinningsList.size()];
for (int i = 0; i < targetPinningsList.size(); i++) {
targetPinnings[i] = targetPinningsList.get(i);
}
return targetPinnings;
}
public static CachePoolStatsProto convert(CachePoolStats stats) {
CachePoolStatsProto.Builder builder = CachePoolStatsProto.newBuilder();
builder.setBytesNeeded(stats.getBytesNeeded());
builder.setBytesCached(stats.getBytesCached());
builder.setBytesOverlimit(stats.getBytesOverlimit());
builder.setFilesNeeded(stats.getFilesNeeded());
builder.setFilesCached(stats.getFilesCached());
return builder.build();
}
public static CachePoolEntryProto convert(CachePoolEntry entry) {
CachePoolEntryProto.Builder builder = CachePoolEntryProto.newBuilder();
builder.setInfo(convert(entry.getInfo()));
builder.setStats(convert(entry.getStats()));
return builder.build();
}
public static DatanodeLocalInfoProto convert(DatanodeLocalInfo info) {
DatanodeLocalInfoProto.Builder builder =
DatanodeLocalInfoProto.newBuilder();
builder.setSoftwareVersion(info.getSoftwareVersion());
builder.setConfigVersion(info.getConfigVersion());
builder.setUptime(info.getUptime());
return builder.build();
}
public static GetAclStatusResponseProto convert(AclStatus e) {
AclStatusProto.Builder builder = AclStatusProto.newBuilder();
builder.setOwner(e.getOwner())
.setGroup(e.getGroup()).setSticky(e.isStickyBit())
.addAllEntries(convertAclEntryProto(e.getEntries()));
if (e.getPermission() != null) {
builder.setPermission(convert(e.getPermission()));
}
AclStatusProto r = builder.build();
return GetAclStatusResponseProto.newBuilder().setResult(r).build();
}
public static EnumSet<XAttrSetFlag> convert(int flag) {
EnumSet<XAttrSetFlag> result =
EnumSet.noneOf(XAttrSetFlag.class);
if ((flag & XAttrSetFlagProto.XATTR_CREATE_VALUE) ==
XAttrSetFlagProto.XATTR_CREATE_VALUE) {
result.add(XAttrSetFlag.CREATE);
}
if ((flag & XAttrSetFlagProto.XATTR_REPLACE_VALUE) ==
XAttrSetFlagProto.XATTR_REPLACE_VALUE) {
result.add(XAttrSetFlag.REPLACE);
}
return result;
}
public static XAttr convertXAttr(XAttrProto a) {
XAttr.Builder builder = new XAttr.Builder();
builder.setNameSpace(convert(a.getNamespace()));
if (a.hasName()) {
builder.setName(a.getName());
}
if (a.hasValue()) {
builder.setValue(a.getValue().toByteArray());
}
return builder.build();
}
public static GetXAttrsResponseProto convertXAttrsResponse(
List<XAttr> xAttrs) {
GetXAttrsResponseProto.Builder builder = GetXAttrsResponseProto
.newBuilder();
if (xAttrs != null) {
builder.addAllXAttrs(convertXAttrProto(xAttrs));
}
return builder.build();
}
public static ListXAttrsResponseProto convertListXAttrsResponse(
List<XAttr> names) {
ListXAttrsResponseProto.Builder builder =
ListXAttrsResponseProto.newBuilder();
if (names != null) {
builder.addAllXAttrs(convertXAttrProto(names));
}
return builder.build();
}
public static EncryptionZoneProto convert(EncryptionZone zone) {
return EncryptionZoneProto.newBuilder()
.setId(zone.getId())
.setPath(zone.getPath())
.setSuite(convert(zone.getSuite()))
.setCryptoProtocolVersion(convert(zone.getVersion()))
.setKeyName(zone.getKeyName())
.build();
}
public static SlotId convert(ShortCircuitShmSlotProto slotId) {
return new SlotId(convert(slotId.getShmId()),
slotId.getSlotIdx());
}
public static GetEditsFromTxidResponseProto convertEditsResponse(
EventBatchList el) {
InotifyProtos.EventsListProto.Builder builder =
InotifyProtos.EventsListProto.newBuilder();
for (EventBatch b : el.getBatches()) {
List<InotifyProtos.EventProto> events = Lists.newArrayList();
for (Event e : b.getEvents()) {
switch (e.getEventType()) {
case CLOSE:
Event.CloseEvent ce = (Event.CloseEvent) e;
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_CLOSE)
.setContents(
InotifyProtos.CloseEventProto.newBuilder()
.setPath(ce.getPath())
.setFileSize(ce.getFileSize())
.setTimestamp(ce.getTimestamp()).build().toByteString()
).build());
break;
case CREATE:
Event.CreateEvent ce2 = (Event.CreateEvent) e;
InotifyProtos.CreateEventProto.Builder pB =
(InotifyProtos.CreateEventProto.newBuilder());
pB.setType(createTypeConvert(ce2.getiNodeType()))
.setPath(ce2.getPath())
.setCtime(ce2.getCtime())
.setOwnerName(ce2.getOwnerName())
.setGroupName(ce2.getGroupName())
.setPerms(convert(ce2.getPerms()))
.setReplication(ce2.getReplication())
.setSymlinkTarget(ce2.getSymlinkTarget() == null ?
"" : ce2.getSymlinkTarget())
.setDefaultBlockSize(ce2.getDefaultBlockSize())
.setOverwrite(ce2.getOverwrite());
if (ce2.isErasureCoded().isPresent()) {
pB.setErasureCoded(ce2.isErasureCoded().get());
}
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_CREATE)
.setContents(pB.build().toByteString())
.build());
break;
case METADATA:
Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
InotifyProtos.MetadataUpdateEventProto.Builder metaB =
InotifyProtos.MetadataUpdateEventProto.newBuilder()
.setPath(me.getPath())
.setType(metadataUpdateTypeConvert(me.getMetadataType()))
.setMtime(me.getMtime())
.setAtime(me.getAtime())
.setReplication(me.getReplication())
.setOwnerName(me.getOwnerName() == null ? "" :
me.getOwnerName())
.setGroupName(me.getGroupName() == null ? "" :
me.getGroupName())
.addAllAcls(me.getAcls() == null ?
Lists.<AclEntryProto>newArrayList() :
convertAclEntryProto(me.getAcls()))
.addAllXAttrs(me.getxAttrs() == null ?
Lists.<XAttrProto>newArrayList() :
convertXAttrProto(me.getxAttrs()))
.setXAttrsRemoved(me.isxAttrsRemoved());
if (me.getPerms() != null) {
metaB.setPerms(convert(me.getPerms()));
}
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_METADATA)
.setContents(metaB.build().toByteString())
.build());
break;
case RENAME:
Event.RenameEvent re = (Event.RenameEvent) e;
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_RENAME)
.setContents(
InotifyProtos.RenameEventProto.newBuilder()
.setSrcPath(re.getSrcPath())
.setDestPath(re.getDstPath())
.setTimestamp(re.getTimestamp()).build().toByteString()
).build());
break;
case APPEND:
Event.AppendEvent re2 = (Event.AppendEvent) e;
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_APPEND)
.setContents(InotifyProtos.AppendEventProto.newBuilder()
.setPath(re2.getPath())
.setNewBlock(re2.toNewBlock()).build().toByteString())
.build());
break;
case UNLINK:
Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_UNLINK)
.setContents(
InotifyProtos.UnlinkEventProto.newBuilder()
.setPath(ue.getPath())
.setTimestamp(ue.getTimestamp()).build().toByteString()
).build());
break;
case TRUNCATE:
Event.TruncateEvent te = (Event.TruncateEvent) e;
events.add(InotifyProtos.EventProto.newBuilder()
.setType(InotifyProtos.EventType.EVENT_TRUNCATE)
.setContents(
InotifyProtos.TruncateEventProto.newBuilder()
.setPath(te.getPath())
.setFileSize(te.getFileSize())
.setTimestamp(te.getTimestamp()).build().toByteString()
).build());
break;
default:
throw new RuntimeException("Unexpected inotify event: " + e);
}
}
builder.addBatch(InotifyProtos.EventBatchProto.newBuilder().
setTxid(b.getTxid()).
addAllEvents(events));
}
builder.setFirstTxid(el.getFirstTxid());
builder.setLastTxid(el.getLastTxid());
builder.setSyncTxid(el.getSyncTxid());
return GetEditsFromTxidResponseProto.newBuilder().setEventsList(
builder.build()).build();
}
public static CryptoProtocolVersion[] convertCryptoProtocolVersions(
List<CryptoProtocolVersionProto> protos) {
List<CryptoProtocolVersion> versions =
Lists.newArrayListWithCapacity(protos.size());
for (CryptoProtocolVersionProto p: protos) {
versions.add(convert(p));
}
return versions.toArray(new CryptoProtocolVersion[]{});
}
public static HdfsProtos.PerFileEncryptionInfoProto convertPerFileEncInfo(
FileEncryptionInfo info) {
if (info == null) {
return null;
}
return HdfsProtos.PerFileEncryptionInfoProto.newBuilder()
.setKey(getByteString(info.getEncryptedDataEncryptionKey()))
.setIv(getByteString(info.getIV()))
.setEzKeyVersionName(info.getEzKeyVersionName())
.build();
}
public static ZoneEncryptionInfoProto convert(CipherSuite suite,
CryptoProtocolVersion version, String keyName) {
return convert(suite, version, keyName, null);
}
public static ZoneEncryptionInfoProto convert(CipherSuite suite,
CryptoProtocolVersion version, String keyName,
ReencryptionInfoProto proto) {
if (suite == null || version == null || keyName == null) {
return null;
}
ZoneEncryptionInfoProto.Builder builder =
ZoneEncryptionInfoProto.newBuilder().setSuite(convert(suite))
.setCryptoProtocolVersion(convert(version)).setKeyName(keyName);
if (proto != null) {
builder.setReencryptionProto(proto);
}
return builder.build();
}
public static FileEncryptionInfo convert(
HdfsProtos.PerFileEncryptionInfoProto fileProto,
CipherSuite suite, CryptoProtocolVersion version, String keyName) {
if (fileProto == null || suite == null || version == null ||
keyName == null) {
return null;
}
byte[] key = fileProto.getKey().toByteArray();
byte[] iv = fileProto.getIv().toByteArray();
String ezKeyVersionName = fileProto.getEzKeyVersionName();
return new FileEncryptionInfo(suite, version, key, iv, keyName,
ezKeyVersionName);
}
public static ReencryptionInfoProto convert(String ezkvn, Long submissionTime,
boolean isCanceled, long numReencrypted, long numFailures,
Long completionTime, String lastFile) {
if (ezkvn == null || submissionTime == null) {
return null;
}
ReencryptionInfoProto.Builder builder =
ReencryptionInfoProto.newBuilder().setEzKeyVersionName(ezkvn)
.setSubmissionTime(submissionTime).setCanceled(isCanceled)
.setNumReencrypted(numReencrypted).setNumFailures(numFailures);
if (completionTime != null) {
builder.setCompletionTime(completionTime);
}
if (lastFile != null) {
builder.setLastFile(lastFile);
}
return builder.build();
}
public static ZoneReencryptionStatusProto convert(ZoneReencryptionStatus zs) {
ZoneReencryptionStatusProto.Builder builder =
ZoneReencryptionStatusProto.newBuilder()
.setId(zs.getId())
.setPath(zs.getZoneName())
.setEzKeyVersionName(zs.getEzKeyVersionName())
.setSubmissionTime(zs.getSubmissionTime())
.setCanceled(zs.isCanceled())
.setNumReencrypted(zs.getFilesReencrypted())
.setNumFailures(zs.getNumReencryptionFailures());
switch (zs.getState()) {
case Submitted:
builder.setState(ReencryptionStateProto.SUBMITTED);
break;
case Processing:
builder.setState(ReencryptionStateProto.PROCESSING);
break;
case Completed:
builder.setState(ReencryptionStateProto.COMPLETED);
break;
default:
throw new IllegalArgumentException("Unknown state " + zs.getState());
}
final long completion = zs.getCompletionTime();
if (completion != 0) {
builder.setCompletionTime(completion);
}
final String file = zs.getLastCheckpointFile();
if (file != null) {
builder.setLastFile(file);
}
return builder.build();
}
public static ZoneReencryptionStatus convert(
ZoneReencryptionStatusProto proto) {
ZoneReencryptionStatus.State state;
switch (proto.getState()) {
case SUBMITTED:
state = ZoneReencryptionStatus.State.Submitted;
break;
case PROCESSING:
state = ZoneReencryptionStatus.State.Processing;
break;
case COMPLETED:
state = ZoneReencryptionStatus.State.Completed;
break;
default:
throw new IllegalArgumentException("Unknown state " + proto.getState());
}
ZoneReencryptionStatus.Builder builder = new ZoneReencryptionStatus.
Builder().
id(proto.getId()).zoneName(proto.getPath()).state(state)
.ezKeyVersionName(proto.getEzKeyVersionName())
.submissionTime(proto.getSubmissionTime()).canceled(proto.getCanceled())
.filesReencrypted(proto.getNumReencrypted())
.fileReencryptionFailures(proto.getNumFailures());
if (proto.hasCompletionTime()) {
builder.completionTime(proto.getCompletionTime());
}
if (proto.hasLastFile()) {
builder.lastCheckpointFile(proto.getLastFile());
}
return builder.build();
}
public static DatanodeInfo[] convert(DatanodeInfosProto datanodeInfosProto) {
List<DatanodeInfoProto> proto = datanodeInfosProto.getDatanodesList();
DatanodeInfo[] infos = new DatanodeInfo[proto.size()];
for (int i = 0; i < infos.length; i++) {
infos[i] = convert(proto.get(i));
}
return infos;
}
static List<DatanodeInfosProto> convert(DatanodeInfo[][] targets) {
DatanodeInfosProto[] ret = new DatanodeInfosProto[targets.length];
for (int i = 0; i < targets.length; i++) {
ret[i] = DatanodeInfosProto.newBuilder()
.addAllDatanodes(convert(targets[i])).build();
}
return Arrays.asList(ret);
}
public static ECSchema convertECSchema(HdfsProtos.ECSchemaProto schema) {
List<HdfsProtos.ECSchemaOptionEntryProto> optionsList =
schema.getOptionsList();
Map<String, String> options = new HashMap<>(optionsList.size());
for (HdfsProtos.ECSchemaOptionEntryProto option : optionsList) {
options.put(option.getKey(), option.getValue());
}
return new ECSchema(schema.getCodecName(), schema.getDataUnits(),
schema.getParityUnits(), options);
}
public static HdfsProtos.ECSchemaProto convertECSchema(ECSchema schema) {
HdfsProtos.ECSchemaProto.Builder builder =
HdfsProtos.ECSchemaProto.newBuilder()
.setCodecName(schema.getCodecName())
.setDataUnits(schema.getNumDataUnits())
.setParityUnits(schema.getNumParityUnits());
Set<Map.Entry<String, String>> entrySet =
schema.getExtraOptions().entrySet();
for (Map.Entry<String, String> entry : entrySet) {
builder.addOptions(HdfsProtos.ECSchemaOptionEntryProto.newBuilder()
.setKey(entry.getKey()).setValue(entry.getValue()).build());
}
return builder.build();
}
public static ErasureCodingPolicyState convertECState(
HdfsProtos.ErasureCodingPolicyState state) {
return ErasureCodingPolicyState.fromValue(state.getNumber());
}
public static HdfsProtos.ErasureCodingPolicyState convertECState(
ErasureCodingPolicyState state) {
return HdfsProtos.ErasureCodingPolicyState.forNumber(state.getValue());
}
/**
* Convert the protobuf to a {@link ErasureCodingPolicy}.
*/
public static ErasureCodingPolicy convertErasureCodingPolicy(
ErasureCodingPolicyProto proto) {
final byte id = (byte) (proto.getId() & 0xFF);
ErasureCodingPolicy policy = SystemErasureCodingPolicies.getByID(id);
if (policy == null) {
// If it's not a built-in policy, populate from the optional PB fields.
// The optional fields are required in this case.
Preconditions.checkArgument(proto.hasName(),
"Missing name field in ErasureCodingPolicy proto");
Preconditions.checkArgument(proto.hasSchema(),
"Missing schema field in ErasureCodingPolicy proto");
Preconditions.checkArgument(proto.hasCellSize(),
"Missing cellsize field in ErasureCodingPolicy proto");
return new ErasureCodingPolicy(proto.getName(),
convertECSchema(proto.getSchema()),
proto.getCellSize(), id);
}
return policy;
}
/**
* Convert the protobuf to a {@link ErasureCodingPolicyInfo}. This should only
* be needed when the caller is interested in the state of the policy.
*/
public static ErasureCodingPolicyInfo convertErasureCodingPolicyInfo(
ErasureCodingPolicyProto proto) {
ErasureCodingPolicy policy = convertErasureCodingPolicy(proto);
ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy);
Preconditions.checkArgument(proto.hasState(),
"Missing state field in ErasureCodingPolicy proto");
info.setState(convertECState(proto.getState()));
return info;
}
private static ErasureCodingPolicyProto.Builder createECPolicyProtoBuilder(
ErasureCodingPolicy policy) {
final ErasureCodingPolicyProto.Builder builder =
ErasureCodingPolicyProto.newBuilder().setId(policy.getId());
// If it's not a built-in policy, need to set the optional fields.
if (SystemErasureCodingPolicies.getByID(policy.getId()) == null) {
builder.setName(policy.getName())
.setSchema(convertECSchema(policy.getSchema()))
.setCellSize(policy.getCellSize());
}
return builder;
}
/**
* Convert a {@link ErasureCodingPolicy} to protobuf.
* This means no state of the policy will be set on the protobuf.
*/
public static ErasureCodingPolicyProto convertErasureCodingPolicy(
ErasureCodingPolicy policy) {
return createECPolicyProtoBuilder(policy).build();
}
/**
* Convert a {@link ErasureCodingPolicyInfo} to protobuf.
* The protobuf will have the policy, and state. State is relevant when:
* 1. Persisting a policy to fsimage
* 2. Returning the policy to the RPC call
* {@link DistributedFileSystem#getAllErasureCodingPolicies()}
*/
public static ErasureCodingPolicyProto convertErasureCodingPolicy(
ErasureCodingPolicyInfo info) {
final ErasureCodingPolicyProto.Builder builder =
createECPolicyProtoBuilder(info.getPolicy());
builder.setState(convertECState(info.getState()));
return builder.build();
}
public static CodecProto convertErasureCodingCodec(String codec,
String coders) {
CodecProto.Builder builder = CodecProto.newBuilder()
.setCodec(codec).setCoders(coders);
return builder.build();
}
public static AddErasureCodingPolicyResponseProto
convertAddErasureCodingPolicyResponse(
AddErasureCodingPolicyResponse response) {
AddErasureCodingPolicyResponseProto.Builder builder =
AddErasureCodingPolicyResponseProto.newBuilder()
.setPolicy(convertErasureCodingPolicy(response.getPolicy()))
.setSucceed(response.isSucceed());
if (!response.isSucceed()) {
builder.setErrorMsg(response.getErrorMsg());
}
return builder.build();
}
public static AddErasureCodingPolicyResponse
convertAddErasureCodingPolicyResponse(
AddErasureCodingPolicyResponseProto proto) {
ErasureCodingPolicy policy = convertErasureCodingPolicy(proto.getPolicy());
if (proto.getSucceed()) {
return new AddErasureCodingPolicyResponse(policy);
} else {
return new AddErasureCodingPolicyResponse(policy, proto.getErrorMsg());
}
}
public static HdfsProtos.DatanodeInfosProto convertToProto(
DatanodeInfo[] datanodeInfos) {
HdfsProtos.DatanodeInfosProto.Builder builder =
HdfsProtos.DatanodeInfosProto.newBuilder();
for (DatanodeInfo datanodeInfo : datanodeInfos) {
builder.addDatanodes(PBHelperClient.convert(datanodeInfo));
}
return builder.build();
}
public static ECTopologyVerifierResult convertECTopologyVerifierResultProto(
HdfsProtos.ECTopologyVerifierResultProto resp) {
return new ECTopologyVerifierResult(resp.getIsSupported(),
resp.getResultMessage());
}
public static HdfsProtos.ECTopologyVerifierResultProto convertECTopologyVerifierResult(
ECTopologyVerifierResult resp) {
final HdfsProtos.ECTopologyVerifierResultProto.Builder builder =
HdfsProtos.ECTopologyVerifierResultProto.newBuilder()
.setIsSupported(resp.isSupported())
.setResultMessage(resp.getResultMessage());
return builder.build();
}
public static EnumSet<AddBlockFlag> convertAddBlockFlags(
List<AddBlockFlagProto> addBlockFlags) {
EnumSet<AddBlockFlag> flags =
EnumSet.noneOf(AddBlockFlag.class);
for (AddBlockFlagProto af : addBlockFlags) {
AddBlockFlag flag = AddBlockFlag.valueOf((short)af.getNumber());
if (flag != null) {
flags.add(flag);
}
}
return flags;
}
public static List<AddBlockFlagProto> convertAddBlockFlags(
EnumSet<AddBlockFlag> flags) {
List<AddBlockFlagProto> ret = new ArrayList<>();
for (AddBlockFlag flag : flags) {
AddBlockFlagProto abfp = AddBlockFlagProto.forNumber(flag.getMode());
if (abfp != null) {
ret.add(abfp);
}
}
return ret;
}
public static ProvidedStorageLocation convert(
HdfsProtos.ProvidedStorageLocationProto providedStorageLocationProto) {
if (providedStorageLocationProto == null) {
return null;
}
String path = providedStorageLocationProto.getPath();
long length = providedStorageLocationProto.getLength();
long offset = providedStorageLocationProto.getOffset();
ByteString nonce = providedStorageLocationProto.getNonce();
if (path == null || length == -1 || offset == -1 || nonce == null) {
return null;
} else {
return new ProvidedStorageLocation(new Path(path), offset, length,
nonce.toByteArray());
}
}
public static HdfsProtos.ProvidedStorageLocationProto convert(
ProvidedStorageLocation providedStorageLocation) {
String path = providedStorageLocation.getPath().toString();
return HdfsProtos.ProvidedStorageLocationProto.newBuilder()
.setPath(path)
.setLength(providedStorageLocation.getLength())
.setOffset(providedStorageLocation.getOffset())
.setNonce(ByteString.copyFrom(providedStorageLocation.getNonce()))
.build();
}
public static EnumSet<OpenFilesType> convertOpenFileTypes(
List<OpenFilesTypeProto> openFilesTypeProtos) {
EnumSet<OpenFilesType> types = EnumSet.noneOf(OpenFilesType.class);
for (OpenFilesTypeProto af : openFilesTypeProtos) {
OpenFilesType type = OpenFilesType.valueOf((short)af.getNumber());
if (type != null) {
types.add(type);
}
}
return types;
}
public static List<OpenFilesTypeProto> convertOpenFileTypes(
EnumSet<OpenFilesType> types) {
List<OpenFilesTypeProto> typeProtos = new ArrayList<>();
for (OpenFilesType type : types) {
OpenFilesTypeProto typeProto = OpenFilesTypeProto
.forNumber(type.getMode());
if (typeProto != null) {
typeProtos.add(typeProto);
}
}
return typeProtos;
}
}
相关信息
相关文章
hadoop ClientDatanodeProtocolPB 源码
hadoop ClientDatanodeProtocolTranslatorPB 源码
hadoop ClientNamenodeProtocolPB 源码
hadoop ClientNamenodeProtocolTranslatorPB 源码
hadoop ReconfigurationProtocolPB 源码
0
赞
热门推荐
-
2、 - 优质文章
-
3、 gate.io
-
6、 golang
-
7、 openharmony