You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/09/19 07:16:09 UTC
[flink] 01/06: [FLINK-10365] [s3] Factor out Hadoop FS classes into
pre-shaded artifact and update to Hadoop 3
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit b0e5642c93e90e31b160dbcd3b93ee0b71757c36
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Fri Aug 24 18:33:47 2018 +0200
[FLINK-10365] [s3] Factor out Hadoop FS classes into pre-shaded artifact and update to Hadoop 3
---
flink-filesystems/flink-fs-hadoop-shaded/README.md | 32 +
flink-filesystems/flink-fs-hadoop-shaded/pom.xml | 262 ++
.../java/org/apache/hadoop/conf/Configuration.java | 3548 ++++++++++++++++++++
.../org/apache/hadoop/util/NativeCodeLoader.java | 58 +
.../java/org/apache/hadoop/util/VersionInfo.java | 133 +
.../src/main/resources/core-default-shaded.xml | 2775 +++++++++++++++
.../src/test/resources/core-site.xml | 2775 +++++++++++++++
flink-filesystems/pom.xml | 6 +-
tools/maven/suppressions.xml | 3 +
9 files changed, 9591 insertions(+), 1 deletion(-)
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/README.md b/flink-filesystems/flink-fs-hadoop-shaded/README.md
new file mode 100644
index 0000000..94ae62a
--- /dev/null
+++ b/flink-filesystems/flink-fs-hadoop-shaded/README.md
@@ -0,0 +1,32 @@
+This project bundles the minimal dependencies from Hadoop's
+FileSystem abstraction and shades them to avoid dependency conflicts.
+
+This project is the basis for the bundled File System adapters
+that are based on Hadoop code, but keep the appearance of Flink
+being Hadoop-free, from a dependency perspective.
+
+For this to work, however, we needed to adapt Hadoop's `Configuration`
+class to load a (shaded) `core-default-shaded.xml` configuration with the
+relocated class names of classes loaded via reflection.
+
+# Changing the Hadoop Version
+
+If you want to change the Hadoop version this project depends on, the following
+steps are required to keep the shading correct:
+
+1. from the respective Hadoop jar (currently 3.1.0),
+ - copy `org/apache/hadoop/conf/Configuration.java` to `src/main/java/org/apache/hadoop/conf/` and
+ - replace `core-default.xml` with `core-default-shaded.xml`.
+ - copy `org/apache/hadoop/util/NativeCodeLoader.java` to `src/main/java/org/apache/hadoop/util/` and
+ - replace the native method stubs as in the current setup (empty methods, or return false)
+ - copy `core-default.xml` to `src/main/resources/core-default-shaded.xml` and
+ - change every occurrence of `org.apache.hadoop` into `org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop`
+ - copy `core-site.xml` to `src/test/resources/core-site.xml` (as is)
+
+2. verify the shaded jar:
+ - does not contain any unshaded classes
+ - all other classes should be under `org.apache.flink.fs.shaded.hadoop3`
+ - there should be a `META-INF/services/org.apache.flink.core.fs.FileSystemFactory` file pointing to two classes: `org.apache.flink.fs.s3hadoop.S3FileSystemFactory` and `org.apache.flink.fs.s3hadoop.S3AFileSystemFactory`
+ - other service files under `META-INF/services` should have their names and contents in the relocated `org.apache.flink.fs.s3hadoop.shaded` package
+ - contains a `core-default-shaded.xml` file
+ - does not contain a `core-default.xml` or `core-site.xml` file
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/pom.xml b/flink-filesystems/flink-fs-hadoop-shaded/pom.xml
new file mode 100644
index 0000000..b8029a1
--- /dev/null
+++ b/flink-filesystems/flink-fs-hadoop-shaded/pom.xml
@@ -0,0 +1,262 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-filesystems</artifactId>
+ <version>1.7-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-fs-hadoop-shaded</artifactId>
+ <name>flink-filesystems :: flink-fs-hadoop-shaded</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <!-- The Hadoop file system abstraction -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${fs.hadoopshaded.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kerby</groupId>
+ <artifactId>kerb-simplekdc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>nimbus-jose-jwt</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+
+ <!-- this is merely an intermediate build artifact and should not be -->
+ <!-- deployed to maven central -->
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- publish the core-site.xml for tests -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- relocate all dependencies to hide them -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <!-- we shade only the parts that are internal to Hadoop and not used / exposed downstream -->
+ <relocation>
+ <pattern>com.google.re2j</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.google.re2j</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.htrace</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.htrace</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.fasterxml</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.codehaus</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.codehaus</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.ctc</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.com.ctc</shadedPattern>
+ </relocation>
+ </relocations>
+ <filters>
+ <!-- remove the classes from Hadoop that we replace with our overwritten implementation -->
+ <filter>
+ <artifact>org.apache.hadoop:hadoop-common</artifact>
+ <excludes>
+ <exclude>org/apache/hadoop/conf/Configuration**</exclude>
+ <exclude>org/apache/hadoop/util/NativeCodeLoader**</exclude>
+ <exclude>org/apache/hadoop/util/VersionInfo**</exclude>
+ <exclude>core-default.xml</exclude>
+ <exclude>common-version-info.properties</exclude>
+ <exclude>org.apache.hadoop.application-classloader.properties</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>properties.dtd</exclude>
+ <exclude>PropertyList-1.0.dtd</exclude>
+ <exclude>META-INF/maven/**</exclude>
+ <exclude>META-INF/services/javax.xml.stream.*</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/conf/Configuration.java b/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/conf/Configuration.java
new file mode 100644
index 0000000..49c81ef
--- /dev/null
+++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -0,0 +1,3548 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import com.ctc.wstx.api.ReaderConfig;
+import com.ctc.wstx.io.StreamBootstrapper;
+import com.ctc.wstx.io.SystemId;
+import com.ctc.wstx.stax.WstxInputFactory;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.BufferedInputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.Writer;
+import java.lang.ref.WeakReference;
+import java.net.InetSocketAddress;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.collections.map.UnmodifiableMap;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringInterner;
+import org.apache.hadoop.util.StringUtils;
+import org.codehaus.stax2.XMLStreamReader2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+
+import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+//
+// This class is copied from the Hadoop Source Code (Apache License 2.0)
+// in order to override config keys to support shading
+//
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+
+@SuppressWarnings("all")
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class Configuration implements Iterable<Map.Entry<String,String>>,
+ Writable {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(Configuration.class);
+
+ private static final Logger LOG_DEPRECATION =
+ LoggerFactory.getLogger(
+ "org.apache.hadoop.conf.Configuration.deprecation");
+
+ private boolean quietmode = true;
+
+ private static final String DEFAULT_STRING_CHECK =
+ "testingforemptydefaultvalue";
+
+ private static boolean restrictSystemPropsDefault = false;
+ private boolean restrictSystemProps = restrictSystemPropsDefault;
+ private boolean allowNullValueProperties = false;
+
+ private static class Resource {
+ private final Object resource;
+ private final String name;
+ private final boolean restrictParser;
+
+ public Resource(Object resource) {
+ this(resource, resource.toString());
+ }
+
+ public Resource(Object resource, boolean useRestrictedParser) {
+ this(resource, resource.toString(), useRestrictedParser);
+ }
+
+ public Resource(Object resource, String name) {
+ this(resource, name, getRestrictParserDefault(resource));
+ }
+
+ public Resource(Object resource, String name, boolean restrictParser) {
+ this.resource = resource;
+ this.name = name;
+ this.restrictParser = restrictParser;
+ }
+
+ public String getName(){
+ return name;
+ }
+
+ public Object getResource() {
+ return resource;
+ }
+
+ public boolean isParserRestricted() {
+ return restrictParser;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ private static boolean getRestrictParserDefault(Object resource) {
+ if (resource instanceof String) {
+ return false;
+ }
+ UserGroupInformation user;
+ try {
+ user = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to determine current user", e);
+ }
+ return user.getRealUser() != null;
+ }
+ }
+
+ /**
+ * List of configuration resources.
+ */
+ private ArrayList<Resource> resources = new ArrayList<Resource>();
+
+ /**
+ * The value reported as the setting resource when a key is set
+ * by code rather than a file resource by dumpConfiguration.
+ */
+ static final String UNKNOWN_RESOURCE = "Unknown";
+
+
+ /**
+ * List of configuration parameters marked <b>final</b>.
+ */
+ private Set<String> finalParameters = Collections.newSetFromMap(
+ new ConcurrentHashMap<String, Boolean>());
+
+ private boolean loadDefaults = true;
+
+ /**
+ * Configuration objects
+ */
+ private static final WeakHashMap<Configuration,Object> REGISTRY =
+ new WeakHashMap<Configuration,Object>();
+
+ /**
+ * List of default Resources. Resources are loaded in the order of the list
+ * entries
+ */
+ private static final CopyOnWriteArrayList<String> defaultResources =
+ new CopyOnWriteArrayList<String>();
+
+ private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>>
+ CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>();
+
+ /**
+ * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
+ */
+ private static final Class<?> NEGATIVE_CACHE_SENTINEL =
+ NegativeCacheSentinel.class;
+
+ /**
+ * Stores the mapping of key to the resource which modifies or loads
+ * the key most recently. Created lazily to avoid wasting memory.
+ */
+ private volatile Map<String, String[]> updatingResource;
+
+ /**
+ * Specify exact input factory to avoid time finding correct one.
+ * Factory is reusable across un-synchronized threads once initialized
+ */
+ private static final WstxInputFactory XML_INPUT_FACTORY =
+ new WstxInputFactory();
+
+ /**
+ * Class to keep the information about the keys which replace the deprecated
+ * ones.
+ *
+ * This class stores the new keys which replace the deprecated keys and also
+ * gives a provision to have a custom message for each of the deprecated key
+ * that is being replaced. It also provides method to get the appropriate
+ * warning message which can be logged whenever the deprecated key is used.
+ */
+ private static class DeprecatedKeyInfo {
+ private final String[] newKeys;
+ private final String customMessage;
+ private final AtomicBoolean accessed = new AtomicBoolean(false);
+
+ DeprecatedKeyInfo(String[] newKeys, String customMessage) {
+ this.newKeys = newKeys;
+ this.customMessage = customMessage;
+ }
+
+ private final String getWarningMessage(String key) {
+ return getWarningMessage(key, null);
+ }
+
+ /**
+ * Method to provide the warning message. It gives the custom message if
+ * non-null, and default message otherwise.
+ * @param key the associated deprecated key.
+ * @param source the property source.
+ * @return message that is to be logged when a deprecated key is used.
+ */
+ private String getWarningMessage(String key, String source) {
+ String warningMessage;
+ if(customMessage == null) {
+ StringBuilder message = new StringBuilder(key);
+ if (source != null) {
+ message.append(" in " + source);
+ }
+ message.append(" is deprecated. Instead, use ");
+ for (int i = 0; i < newKeys.length; i++) {
+ message.append(newKeys[i]);
+ if(i != newKeys.length-1) {
+ message.append(", ");
+ }
+ }
+ warningMessage = message.toString();
+ }
+ else {
+ warningMessage = customMessage;
+ }
+ return warningMessage;
+ }
+
+ boolean getAndSetAccessed() {
+ return accessed.getAndSet(true);
+ }
+
+ public void clearAccessed() {
+ accessed.set(false);
+ }
+ }
+
+ /**
+ * A pending addition to the global set of deprecated keys.
+ */
+ public static class DeprecationDelta {
+ private final String key;
+ private final String[] newKeys;
+ private final String customMessage;
+
+ DeprecationDelta(String key, String[] newKeys, String customMessage) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(newKeys);
+ Preconditions.checkArgument(newKeys.length > 0);
+ this.key = key;
+ this.newKeys = newKeys;
+ this.customMessage = customMessage;
+ }
+
+ public DeprecationDelta(String key, String newKey, String customMessage) {
+ this(key, new String[] { newKey }, customMessage);
+ }
+
+ public DeprecationDelta(String key, String newKey) {
+ this(key, new String[] { newKey }, null);
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public String[] getNewKeys() {
+ return newKeys;
+ }
+
+ public String getCustomMessage() {
+ return customMessage;
+ }
+ }
+
+ /**
+ * The set of all keys which are deprecated.
+ *
+ * DeprecationContext objects are immutable.
+ */
+ private static class DeprecationContext {
+ /**
+ * Stores the deprecated keys, the new keys which replace the deprecated keys
+ * and custom message(if any provided).
+ */
+ private final Map<String, DeprecatedKeyInfo> deprecatedKeyMap;
+
+ /**
+ * Stores a mapping from superseding keys to the keys which they deprecate.
+ */
+ private final Map<String, String> reverseDeprecatedKeyMap;
+
+ /**
+ * Create a new DeprecationContext by copying a previous DeprecationContext
+ * and adding some deltas.
+ *
+ * @param other The previous deprecation context to copy, or null to start
+ * from nothing.
+ * @param deltas The deltas to apply.
+ */
+ @SuppressWarnings("unchecked")
+ DeprecationContext(DeprecationContext other, DeprecationDelta[] deltas) {
+ HashMap<String, DeprecatedKeyInfo> newDeprecatedKeyMap =
+ new HashMap<String, DeprecatedKeyInfo>();
+ HashMap<String, String> newReverseDeprecatedKeyMap =
+ new HashMap<String, String>();
+ if (other != null) {
+ for (Entry<String, DeprecatedKeyInfo> entry :
+ other.deprecatedKeyMap.entrySet()) {
+ newDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
+ }
+ for (Entry<String, String> entry :
+ other.reverseDeprecatedKeyMap.entrySet()) {
+ newReverseDeprecatedKeyMap.put(entry.getKey(), entry.getValue());
+ }
+ }
+ for (DeprecationDelta delta : deltas) {
+ if (!newDeprecatedKeyMap.containsKey(delta.getKey())) {
+ DeprecatedKeyInfo newKeyInfo =
+ new DeprecatedKeyInfo(delta.getNewKeys(), delta.getCustomMessage());
+ newDeprecatedKeyMap.put(delta.key, newKeyInfo);
+ for (String newKey : delta.getNewKeys()) {
+ newReverseDeprecatedKeyMap.put(newKey, delta.key);
+ }
+ }
+ }
+ this.deprecatedKeyMap =
+ UnmodifiableMap.decorate(newDeprecatedKeyMap);
+ this.reverseDeprecatedKeyMap =
+ UnmodifiableMap.decorate(newReverseDeprecatedKeyMap);
+ }
+
+ Map<String, DeprecatedKeyInfo> getDeprecatedKeyMap() {
+ return deprecatedKeyMap;
+ }
+
+ Map<String, String> getReverseDeprecatedKeyMap() {
+ return reverseDeprecatedKeyMap;
+ }
+ }
+
+ private static DeprecationDelta[] defaultDeprecations =
+ new DeprecationDelta[] {
+ new DeprecationDelta("topology.script.file.name",
+ CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY),
+ new DeprecationDelta("topology.script.number.args",
+ CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY),
+ new DeprecationDelta("hadoop.configured.node.mapping",
+ CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY),
+ new DeprecationDelta("topology.node.switch.mapping.impl",
+ CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY),
+ new DeprecationDelta("dfs.df.interval",
+ CommonConfigurationKeys.FS_DF_INTERVAL_KEY),
+ new DeprecationDelta("fs.default.name",
+ CommonConfigurationKeys.FS_DEFAULT_NAME_KEY),
+ new DeprecationDelta("dfs.umaskmode",
+ CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY),
+ new DeprecationDelta("dfs.nfs.exports.allowed.hosts",
+ CommonConfigurationKeys.NFS_EXPORTS_ALLOWED_HOSTS_KEY)
+ };
+
+ /**
+ * The global DeprecationContext.
+ */
+ private static AtomicReference<DeprecationContext> deprecationContext =
+ new AtomicReference<DeprecationContext>(
+ new DeprecationContext(null, defaultDeprecations));
+
+ /**
+ * Adds a set of deprecated keys to the global deprecations.
+ *
+ * This method is lockless. It works by means of creating a new
+ * DeprecationContext based on the old one, and then atomically swapping in
+ * the new context. If someone else updated the context in between us reading
+ * the old context and swapping in the new one, we try again until we win the
+ * race.
+ *
+ * @param deltas The deprecations to add.
+ */
+ public static void addDeprecations(DeprecationDelta[] deltas) {
+ DeprecationContext prev, next;
+ do {
+ prev = deprecationContext.get();
+ next = new DeprecationContext(prev, deltas);
+ } while (!deprecationContext.compareAndSet(prev, next));
+ }
+
+ /**
+ * Adds the deprecated key to the global deprecation map.
+ * It does not override any existing entries in the deprecation map.
+ * This is to be used only by the developers in order to add deprecation of
+ * keys, and attempts to call this method after loading resources once,
+ * would lead to <tt>UnsupportedOperationException</tt>
+ *
+ * If a key is deprecated in favor of multiple keys, they are all treated as
+ * aliases of each other, and setting any one of them resets all the others
+ * to the new value.
+ *
+ * If you have multiple deprecation entries to add, it is more efficient to
+ * use #addDeprecations(DeprecationDelta[] deltas) instead.
+ *
+ * @param key
+ * @param newKeys
+ * @param customMessage
+ * @deprecated use {@link #addDeprecation(String key, String newKey,
+ String customMessage)} instead
+ */
+ @Deprecated
+ public static void addDeprecation(String key, String[] newKeys,
+ String customMessage) {
+ addDeprecations(new DeprecationDelta[] {
+ new DeprecationDelta(key, newKeys, customMessage)
+ });
+ }
+
+ /**
+ * Adds the deprecated key to the global deprecation map.
+ * It does not override any existing entries in the deprecation map.
+ * This is to be used only by the developers in order to add deprecation of
+ * keys, and attempts to call this method after loading resources once,
+ * would lead to <tt>UnsupportedOperationException</tt>
+ *
+ * If you have multiple deprecation entries to add, it is more efficient to
+ * use #addDeprecations(DeprecationDelta[] deltas) instead.
+ *
+ * @param key
+ * @param newKey
+ * @param customMessage
+ */
+ public static void addDeprecation(String key, String newKey,
+ String customMessage) {
+ addDeprecation(key, new String[] {newKey}, customMessage);
+ }
+
+ /**
+ * Adds the deprecated key to the global deprecation map when no custom
+ * message is provided.
+ * It does not override any existing entries in the deprecation map.
+ * This is to be used only by the developers in order to add deprecation of
+ * keys, and attempts to call this method after loading resources once,
+ * would lead to <tt>UnsupportedOperationException</tt>
+ *
+ * If a key is deprecated in favor of multiple keys, they are all treated as
+ * aliases of each other, and setting any one of them resets all the others
+ * to the new value.
+ *
+ * If you have multiple deprecation entries to add, it is more efficient to
+ * use #addDeprecations(DeprecationDelta[] deltas) instead.
+ *
+ * @param key Key that is to be deprecated
+ * @param newKeys list of keys that take up the values of deprecated key
+ * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
+ */
+ @Deprecated
+ public static void addDeprecation(String key, String[] newKeys) {
+ addDeprecation(key, newKeys, null);
+ }
+
+ /**
+ * Adds the deprecated key to the global deprecation map when no custom
+ * message is provided.
+ * It does not override any existing entries in the deprecation map.
+ * This is to be used only by the developers in order to add deprecation of
+ * keys, and attempts to call this method after loading resources once,
+ * would lead to <tt>UnsupportedOperationException</tt>
+ *
+ * If you have multiple deprecation entries to add, it is more efficient to
+ * use #addDeprecations(DeprecationDelta[] deltas) instead.
+ *
+ * @param key Key that is to be deprecated
+ * @param newKey key that takes up the value of deprecated key
+ */
+ public static void addDeprecation(String key, String newKey) {
+ addDeprecation(key, new String[] {newKey}, null);
+ }
+
+ /**
+ * checks whether the given <code>key</code> is deprecated.
+ *
+ * @param key the parameter which is to be checked for deprecation
+ * @return <code>true</code> if the key is deprecated and
+ * <code>false</code> otherwise.
+ */
+ public static boolean isDeprecated(String key) {
+ return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
+ }
+
+ private static String getDeprecatedKey(String key) {
+ return deprecationContext.get().getReverseDeprecatedKeyMap().get(key);
+ }
+
+ private static DeprecatedKeyInfo getDeprecatedKeyInfo(String key) {
+ return deprecationContext.get().getDeprecatedKeyMap().get(key);
+ }
+
+ /**
+ * Sets all deprecated properties that are not currently set but have a
+ * corresponding new property that is set. Useful for iterating the
+ * properties when all deprecated properties for currently set properties
+ * need to be present.
+ */
+ public void setDeprecatedProperties() {
+ DeprecationContext deprecations = deprecationContext.get();
+ Properties props = getProps();
+ Properties overlay = getOverlay();
+ for (Map.Entry<String, DeprecatedKeyInfo> entry :
+ deprecations.getDeprecatedKeyMap().entrySet()) {
+ String depKey = entry.getKey();
+ if (!overlay.contains(depKey)) {
+ for (String newKey : entry.getValue().newKeys) {
+ String val = overlay.getProperty(newKey);
+ if (val != null) {
+ props.setProperty(depKey, val);
+ overlay.setProperty(depKey, val);
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Checks for the presence of the property <code>name</code> in the
+ * deprecation map. Returns the first of the list of new keys if present
+ * in the deprecation map or the <code>name</code> itself. If the property
+ * is not presently set but the property map contains an entry for the
+ * deprecated key, the value of the deprecated key is set as the value for
+ * the provided property name.
+ *
+ * @param deprecations deprecation context
+ * @param name the property name
+ * @return the first property in the list of properties mapping
+ * the <code>name</code> or the <code>name</code> itself.
+ */
+ private String[] handleDeprecation(DeprecationContext deprecations,
+ String name) {
+ if (null != name) {
+ name = name.trim();
+ }
+ // Initialize the return value with requested name
+ String[] names = new String[]{name};
+ // Deprecated keys are logged once and an updated names are returned
+ DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
+ if (keyInfo != null) {
+ if (!keyInfo.getAndSetAccessed()) {
+ logDeprecation(keyInfo.getWarningMessage(name));
+ }
+ // Override return value for deprecated keys
+ names = keyInfo.newKeys;
+ }
+ // If there are no overlay values we can return early
+ Properties overlayProperties = getOverlay();
+ if (overlayProperties.isEmpty()) {
+ return names;
+ }
+ // Update properties and overlays with reverse lookup values
+ for (String n : names) {
+ String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n);
+ if (deprecatedKey != null && !overlayProperties.containsKey(n)) {
+ String deprecatedValue = overlayProperties.getProperty(deprecatedKey);
+ if (deprecatedValue != null) {
+ getProps().setProperty(n, deprecatedValue);
+ overlayProperties.setProperty(n, deprecatedValue);
+ }
+ }
+ }
+ return names;
+ }
+
+ private void handleDeprecation() {
+ LOG.debug("Handling deprecation for all properties in config...");
+ DeprecationContext deprecations = deprecationContext.get();
+ Set<Object> keys = new HashSet<Object>();
+ keys.addAll(getProps().keySet());
+ for (Object item: keys) {
+ LOG.debug("Handling deprecation for " + (String)item);
+ handleDeprecation(deprecations, (String)item);
+ }
+ }
+
+ static {
+ // Add default resources
+ addDefaultResource("core-default-shaded.xml");
+ addDefaultResource("core-site.xml");
+
+ // print deprecation warning if hadoop-site.xml is found in classpath
+ ClassLoader cL = Thread.currentThread().getContextClassLoader();
+ if (cL == null) {
+ cL = Configuration.class.getClassLoader();
+ }
+ if (cL.getResource("hadoop-site.xml") != null) {
+ LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
+ "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
+ + "mapred-site.xml and hdfs-site.xml to override properties of " +
+ "core-default.xml, mapred-default.xml and hdfs-default.xml " +
+ "respectively");
+ addDefaultResource("hadoop-site.xml");
+ }
+ }
+
+ private Properties properties;
+ private Properties overlay;
+ private ClassLoader classLoader;
+ {
+ classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = Configuration.class.getClassLoader();
+ }
+ }
+
+ /** A new configuration. */
+ public Configuration() {
+ this(true);
+ }
+
+ /** A new configuration where the behavior of reading from the default
+ * resources can be turned off.
+ *
+ * If the parameter {@code loadDefaults} is false, the new instance
+ * will not load resources from the default files.
+ * @param loadDefaults specifies whether to load from the default files
+ */
+ public Configuration(boolean loadDefaults) {
+ this.loadDefaults = loadDefaults;
+ synchronized(Configuration.class) {
+ REGISTRY.put(this, null);
+ }
+ }
+
+ /**
+ * A new configuration with the same settings cloned from another.
+ *
+ * @param other the configuration from which to clone settings.
+ */
+ @SuppressWarnings("unchecked")
+ public Configuration(Configuration other) {
+ this.resources = (ArrayList<Resource>) other.resources.clone();
+ synchronized(other) {
+ if (other.properties != null) {
+ this.properties = (Properties)other.properties.clone();
+ }
+
+ if (other.overlay!=null) {
+ this.overlay = (Properties)other.overlay.clone();
+ }
+
+ this.restrictSystemProps = other.restrictSystemProps;
+ if (other.updatingResource != null) {
+ this.updatingResource = new ConcurrentHashMap<String, String[]>(
+ other.updatingResource);
+ }
+ this.finalParameters = Collections.newSetFromMap(
+ new ConcurrentHashMap<String, Boolean>());
+ this.finalParameters.addAll(other.finalParameters);
+ }
+
+ synchronized(Configuration.class) {
+ REGISTRY.put(this, null);
+ }
+ this.classLoader = other.classLoader;
+ this.loadDefaults = other.loadDefaults;
+ setQuietMode(other.getQuietMode());
+ }
+
+ /**
+ * Reload existing configuration instances.
+ */
+ public static synchronized void reloadExistingConfigurations() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reloading " + REGISTRY.keySet().size()
+ + " existing configurations");
+ }
+ for (Configuration conf : REGISTRY.keySet()) {
+ conf.reloadConfiguration();
+ }
+ }
+
+ /**
+ * Add a default resource. Resources are loaded in the order of the resources
+ * added.
+ * @param name file name. File should be present in the classpath.
+ */
+ public static synchronized void addDefaultResource(String name) {
+ if(!defaultResources.contains(name)) {
+ defaultResources.add(name);
+ for(Configuration conf : REGISTRY.keySet()) {
+ if(conf.loadDefaults) {
+ conf.reloadConfiguration();
+ }
+ }
+ }
+ }
+
+ public static void setRestrictSystemPropertiesDefault(boolean val) {
+ restrictSystemPropsDefault = val;
+ }
+
+ public void setRestrictSystemProperties(boolean val) {
+ this.restrictSystemProps = val;
+ }
+
+ /**
+ * Add a configuration resource.
+ *
+ * The properties of this resource will override properties of previously
+ * added resources, unless they were marked <a href="#Final">final</a>.
+ *
+ * @param name resource to be added, the classpath is examined for a file
+ * with that name.
+ */
+ public void addResource(String name) {
+ addResourceObject(new Resource(name));
+ }
+
+ public void addResource(String name, boolean restrictedParser) {
+ addResourceObject(new Resource(name, restrictedParser));
+ }
+
+ /**
+ * Add a configuration resource.
+ *
+ * The properties of this resource will override properties of previously
+ * added resources, unless they were marked <a href="#Final">final</a>.
+ *
+ * @param url url of the resource to be added, the local filesystem is
+ * examined directly to find the resource, without referring to
+ * the classpath.
+ */
+ public void addResource(URL url) {
+ addResourceObject(new Resource(url));
+ }
+
+ public void addResource(URL url, boolean restrictedParser) {
+ addResourceObject(new Resource(url, restrictedParser));
+ }
+
+ /**
+ * Add a configuration resource.
+ *
+ * The properties of this resource will override properties of previously
+ * added resources, unless they were marked <a href="#Final">final</a>.
+ *
+ * @param file file-path of resource to be added, the local filesystem is
+ * examined directly to find the resource, without referring to
+ * the classpath.
+ */
+ public void addResource(Path file) {
+ addResourceObject(new Resource(file));
+ }
+
+ public void addResource(Path file, boolean restrictedParser) {
+ addResourceObject(new Resource(file, restrictedParser));
+ }
+
+ /**
+ * Add a configuration resource.
+ *
+ * The properties of this resource will override properties of previously
+ * added resources, unless they were marked <a href="#Final">final</a>.
+ *
+ * WARNING: The contents of the InputStream will be cached, by this method.
+ * So use this sparingly because it does increase the memory consumption.
+ *
+ * @param in InputStream to deserialize the object from. In will be read from
+ * when a get or set is called next. After it is read the stream will be
+ * closed.
+ */
+ public void addResource(InputStream in) {
+ addResourceObject(new Resource(in));
+ }
+
+ public void addResource(InputStream in, boolean restrictedParser) {
+ addResourceObject(new Resource(in, restrictedParser));
+ }
+
+ /**
+ * Add a configuration resource.
+ *
+ * The properties of this resource will override properties of previously
+ * added resources, unless they were marked <a href="#Final">final</a>.
+ *
+ * @param in InputStream to deserialize the object from.
+ * @param name the name of the resource because InputStream.toString is not
+ * very descriptive some times.
+ */
+ public void addResource(InputStream in, String name) {
+ addResourceObject(new Resource(in, name));
+ }
+
+ public void addResource(InputStream in, String name,
+ boolean restrictedParser) {
+ addResourceObject(new Resource(in, name, restrictedParser));
+ }
+
+ /**
+ * Add a configuration resource.
+ *
+ * The properties of this resource will override properties of previously
+ * added resources, unless they were marked <a href="#Final">final</a>.
+ *
+ * @param conf Configuration object from which to load properties
+ */
+ public void addResource(Configuration conf) {
+ addResourceObject(new Resource(conf.getProps(), conf.restrictSystemProps));
+ }
+
+
+
+ /**
+ * Reload configuration from previously added resources.
+ *
+ * This method will clear all the configuration read from the added
+ * resources, and final parameters. This will make the resources to
+ * be read again before accessing the values. Values that are added
+ * via set methods will overlay values read from the resources.
+ */
+ public synchronized void reloadConfiguration() {
+ properties = null; // trigger reload
+ finalParameters.clear(); // clear site-limits
+ }
+
+ private synchronized void addResourceObject(Resource resource) {
+ resources.add(resource); // add to resources
+ restrictSystemProps |= resource.isParserRestricted();
+ reloadConfiguration();
+ }
+
+ private static final int MAX_SUBST = 20;
+
+ private static final int SUB_START_IDX = 0;
+ private static final int SUB_END_IDX = SUB_START_IDX + 1;
+
+ /**
+ * This is a manual implementation of the following regex
+ * "\\$\\{[^\\}\\$\u0020]+\\}". It can be 15x more efficient than
+ * a regex matcher as demonstrated by HADOOP-11506. This is noticeable with
+ * Hadoop apps building on the assumption Configuration#get is an O(1)
+ * hash table lookup, especially when the eval is a long string.
+ *
+ * @param eval a string that may contain variables requiring expansion.
+ * @return a 2-element int array res such that
+ * eval.substring(res[0], res[1]) is "var" for the left-most occurrence of
+ * ${var} in eval. If no variable is found -1, -1 is returned.
+ */
+ private static int[] findSubVariable(String eval) {
+ int[] result = {-1, -1};
+
+ int matchStart;
+ int leftBrace;
+
+ // scanning for a brace first because it's less frequent than $
+ // that can occur in nested class names
+ //
+ match_loop:
+ for (matchStart = 1, leftBrace = eval.indexOf('{', matchStart);
+ // minimum left brace position (follows '$')
+ leftBrace > 0
+ // right brace of a smallest valid expression "${c}"
+ && leftBrace + "{c".length() < eval.length();
+ leftBrace = eval.indexOf('{', matchStart)) {
+ int matchedLen = 0;
+ if (eval.charAt(leftBrace - 1) == '$') {
+ int subStart = leftBrace + 1; // after '{'
+ for (int i = subStart; i < eval.length(); i++) {
+ switch (eval.charAt(i)) {
+ case '}':
+ if (matchedLen > 0) { // match
+ result[SUB_START_IDX] = subStart;
+ result[SUB_END_IDX] = subStart + matchedLen;
+ break match_loop;
+ }
+ // fall through to skip 1 char
+ case ' ':
+ case '$':
+ matchStart = i + 1;
+ continue match_loop;
+ default:
+ matchedLen++;
+ }
+ }
+ // scanned from "${" to the end of eval, and no reset via ' ', '$':
+ // no match!
+ break match_loop;
+ } else {
+ // not a start of a variable
+ //
+ matchStart = leftBrace + 1;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Attempts to repeatedly expand the value {@code expr} by replacing the
+ * left-most substring of the form "${var}" in the following precedence order
+ * <ol>
+ * <li>by the value of the environment variable "var" if defined</li>
+ * <li>by the value of the Java system property "var" if defined</li>
+ * <li>by the value of the configuration key "var" if defined</li>
+ * </ol>
+ *
+ * If var is unbounded the current state of expansion "prefix${var}suffix" is
+ * returned.
+ * <p>
+ * This function also detects self-referential substitutions, i.e.
+ * <pre>
+ * {@code
+ * foo.bar = ${foo.bar}
+ * }
+ * </pre>
+ * If a cycle is detected then the original expr is returned. Loops
+ * involving multiple substitutions are not detected.
+ *
+ * @param expr the literal value of a config key
+ * @return null if expr is null, otherwise the value resulting from expanding
+ * expr using the algorithm above.
+ * @throws IllegalArgumentException when more than
+ * {@link Configuration#MAX_SUBST} replacements are required
+ */
+ private String substituteVars(String expr) {
+ if (expr == null) {
+ return null;
+ }
+ String eval = expr;
+ for(int s = 0; s < MAX_SUBST; s++) {
+ final int[] varBounds = findSubVariable(eval);
+ if (varBounds[SUB_START_IDX] == -1) {
+ return eval;
+ }
+ final String var = eval.substring(varBounds[SUB_START_IDX],
+ varBounds[SUB_END_IDX]);
+ String val = null;
+ if (!restrictSystemProps) {
+ try {
+ if (var.startsWith("env.") && 4 < var.length()) {
+ String v = var.substring(4);
+ int i = 0;
+ for (; i < v.length(); i++) {
+ char c = v.charAt(i);
+ if (c == ':' && i < v.length() - 1 && v.charAt(i + 1) == '-') {
+ val = getenv(v.substring(0, i));
+ if (val == null || val.length() == 0) {
+ val = v.substring(i + 2);
+ }
+ break;
+ } else if (c == '-') {
+ val = getenv(v.substring(0, i));
+ if (val == null) {
+ val = v.substring(i + 1);
+ }
+ break;
+ }
+ }
+ if (i == v.length()) {
+ val = getenv(v);
+ }
+ } else {
+ val = getProperty(var);
+ }
+ } catch (SecurityException se) {
+ LOG.warn("Unexpected SecurityException in Configuration", se);
+ }
+ }
+ if (val == null) {
+ val = getRaw(var);
+ }
+ if (val == null) {
+ return eval; // return literal ${var}: var is unbound
+ }
+
+ final int dollar = varBounds[SUB_START_IDX] - "${".length();
+ final int afterRightBrace = varBounds[SUB_END_IDX] + "}".length();
+ final String refVar = eval.substring(dollar, afterRightBrace);
+
+ // detect self-referential values
+ if (val.contains(refVar)) {
+ return expr; // return original expression if there is a loop
+ }
+
+ // substitute
+ eval = eval.substring(0, dollar)
+ + val
+ + eval.substring(afterRightBrace);
+ }
+ throw new IllegalStateException("Variable substitution depth too large: "
+ + MAX_SUBST + " " + expr);
+ }
+
+ String getenv(String name) {
+ return System.getenv(name);
+ }
+
+ String getProperty(String key) {
+ return System.getProperty(key);
+ }
+
+ /**
+ * Get the value of the <code>name</code> property, <code>null</code> if
+ * no such property exists. If the key is deprecated, it returns the value of
+ * the first key which replaces the deprecated key and is not null.
+ *
+ * Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ * before being returned.
+ *
+ * @param name the property name, will be trimmed before get value.
+ * @return the value of the <code>name</code> or its replacing property,
+ * or null if no such property exists.
+ */
+ public String get(String name) {
+ String[] names = handleDeprecation(deprecationContext.get(), name);
+ String result = null;
+ for(String n : names) {
+ result = substituteVars(getProps().getProperty(n));
+ }
+ return result;
+ }
+
+ /**
+ * Set Configuration to allow keys without values during setup. Intended
+ * for use during testing.
+ *
+ * @param val If true, will allow Configuration to store keys without values
+ */
+ @VisibleForTesting
+ public void setAllowNullValueProperties( boolean val ) {
+ this.allowNullValueProperties = val;
+ }
+
+ public void setRestrictSystemProps(boolean val) {
+ this.restrictSystemProps = val;
+ }
+
+ /**
+ * Return existence of the <code>name</code> property, but only for
+ * names which have no valid value, usually non-existent or commented
+ * out in XML.
+ *
+ * @param name the property name
+ * @return true if the property <code>name</code> exists without value
+ */
+ @VisibleForTesting
+ public boolean onlyKeyExists(String name) {
+ String[] names = handleDeprecation(deprecationContext.get(), name);
+ for(String n : names) {
+ if ( getProps().getProperty(n,DEFAULT_STRING_CHECK)
+ .equals(DEFAULT_STRING_CHECK) ) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ * <code>null</code> if no such property exists.
+ * If the key is deprecated, it returns the value of
+ * the first key which replaces the deprecated key and is not null
+ *
+ * Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ * before being returned.
+ *
+ * @param name the property name.
+ * @return the value of the <code>name</code> or its replacing property,
+ * or null if no such property exists.
+ */
+ public String getTrimmed(String name) {
+ String value = get(name);
+
+ if (null == value) {
+ return null;
+ } else {
+ return value.trim();
+ }
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ * <code>defaultValue</code> if no such property exists.
+ * See @{Configuration#getTrimmed} for more details.
+ *
+ * @param name the property name.
+ * @param defaultValue the property default value.
+ * @return the value of the <code>name</code> or defaultValue
+ * if it is not set.
+ */
+ public String getTrimmed(String name, String defaultValue) {
+ String ret = getTrimmed(name);
+ return ret == null ? defaultValue : ret;
+ }
+
+ /**
+ * Get the value of the <code>name</code> property, without doing
+ * <a href="#VariableExpansion">variable expansion</a>.If the key is
+ * deprecated, it returns the value of the first key which replaces
+ * the deprecated key and is not null.
+ *
+ * @param name the property name.
+ * @return the value of the <code>name</code> property or
+ * its replacing property and null if no such property exists.
+ */
+ public String getRaw(String name) {
+ String[] names = handleDeprecation(deprecationContext.get(), name);
+ String result = null;
+ for(String n : names) {
+ result = getProps().getProperty(n);
+ }
+ return result;
+ }
+
+ /**
+ * Returns alternative names (non-deprecated keys or previously-set deprecated keys)
+ * for a given non-deprecated key.
+ * If the given key is deprecated, return null.
+ *
+ * @param name property name.
+ * @return alternative names.
+ */
+ private String[] getAlternativeNames(String name) {
+ String altNames[] = null;
+ DeprecatedKeyInfo keyInfo = null;
+ DeprecationContext cur = deprecationContext.get();
+ String depKey = cur.getReverseDeprecatedKeyMap().get(name);
+ if(depKey != null) {
+ keyInfo = cur.getDeprecatedKeyMap().get(depKey);
+ if(keyInfo.newKeys.length > 0) {
+ if(getProps().containsKey(depKey)) {
+ //if deprecated key is previously set explicitly
+ List<String> list = new ArrayList<String>();
+ list.addAll(Arrays.asList(keyInfo.newKeys));
+ list.add(depKey);
+ altNames = list.toArray(new String[list.size()]);
+ }
+ else {
+ altNames = keyInfo.newKeys;
+ }
+ }
+ }
+ return altNames;
+ }
+
+ /**
+ * Set the <code>value</code> of the <code>name</code> property. If
+ * <code>name</code> is deprecated or there is a deprecated name associated to it,
+ * it sets the value to both names. Name will be trimmed before put into
+ * configuration.
+ *
+ * @param name property name.
+ * @param value property value.
+ */
+ public void set(String name, String value) {
+ set(name, value, null);
+ }
+
+ /**
+ * Set the <code>value</code> of the <code>name</code> property. If
+ * <code>name</code> is deprecated, it also sets the <code>value</code> to
+ * the keys that replace the deprecated key. Name will be trimmed before put
+ * into configuration.
+ *
+ * @param name property name.
+ * @param value property value.
+ * @param source the place that this configuration value came from
+ * (For debugging).
+ * @throws IllegalArgumentException when the value or name is null.
+ */
+ public void set(String name, String value, String source) {
+ Preconditions.checkArgument(
+ name != null,
+ "Property name must not be null");
+ Preconditions.checkArgument(
+ value != null,
+ "The value of property %s must not be null", name);
+ name = name.trim();
+ DeprecationContext deprecations = deprecationContext.get();
+ if (deprecations.getDeprecatedKeyMap().isEmpty()) {
+ getProps();
+ }
+ getOverlay().setProperty(name, value);
+ getProps().setProperty(name, value);
+ String newSource = (source == null ? "programmatically" : source);
+
+ if (!isDeprecated(name)) {
+ putIntoUpdatingResource(name, new String[] {newSource});
+ String[] altNames = getAlternativeNames(name);
+ if(altNames != null) {
+ for(String n: altNames) {
+ if(!n.equals(name)) {
+ getOverlay().setProperty(n, value);
+ getProps().setProperty(n, value);
+ putIntoUpdatingResource(n, new String[] {newSource});
+ }
+ }
+ }
+ }
+ else {
+ String[] names = handleDeprecation(deprecationContext.get(), name);
+ String altSource = "because " + name + " is deprecated";
+ for(String n : names) {
+ getOverlay().setProperty(n, value);
+ getProps().setProperty(n, value);
+ putIntoUpdatingResource(n, new String[] {altSource});
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void logDeprecation(String message) {
+ LOG_DEPRECATION.info(message);
+ }
+
+ void logDeprecationOnce(String name, String source) {
+ DeprecatedKeyInfo keyInfo = getDeprecatedKeyInfo(name);
+ if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
+ LOG_DEPRECATION.info(keyInfo.getWarningMessage(name, source));
+ }
+ }
+
+ /**
+ * Unset a previously set property.
+ */
+ public synchronized void unset(String name) {
+ String[] names = null;
+ if (!isDeprecated(name)) {
+ names = getAlternativeNames(name);
+ if(names == null) {
+ names = new String[]{name};
+ }
+ }
+ else {
+ names = handleDeprecation(deprecationContext.get(), name);
+ }
+
+ for(String n: names) {
+ getOverlay().remove(n);
+ getProps().remove(n);
+ }
+ }
+
+ /**
+ * Sets a property if it is currently unset.
+ * @param name the property name
+ * @param value the new value
+ */
+ public synchronized void setIfUnset(String name, String value) {
+ if (get(name) == null) {
+ set(name, value);
+ }
+ }
+
+ private synchronized Properties getOverlay() {
+ if (overlay==null){
+ overlay=new Properties();
+ }
+ return overlay;
+ }
+
+ /**
+ * Get the value of the <code>name</code>. If the key is deprecated,
+ * it returns the value of the first key which replaces the deprecated key
+ * and is not null.
+ * If no such property exists,
+ * then <code>defaultValue</code> is returned.
+ *
+ * @param name property name, will be trimmed before get value.
+ * @param defaultValue default value.
+ * @return property value, or <code>defaultValue</code> if the property
+ * doesn't exist.
+ */
+ public String get(String name, String defaultValue) {
+ String[] names = handleDeprecation(deprecationContext.get(), name);
+ String result = null;
+ for(String n : names) {
+ result = substituteVars(getProps().getProperty(n, defaultValue));
+ }
+ return result;
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as an <code>int</code>.
+ *
+ * If no such property exists, the provided default value is returned,
+ * or if the specified value is not a valid <code>int</code>,
+ * then an error is thrown.
+ *
+ * @param name property name.
+ * @param defaultValue default value.
+ * @throws NumberFormatException when the value is invalid
+ * @return property value as an <code>int</code>,
+ * or <code>defaultValue</code>.
+ */
+ public int getInt(String name, int defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null)
+ return defaultValue;
+ String hexString = getHexDigits(valueString);
+ if (hexString != null) {
+ return Integer.parseInt(hexString, 16);
+ }
+ return Integer.parseInt(valueString);
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a set of comma-delimited
+ * <code>int</code> values.
+ *
+ * If no such property exists, an empty array is returned.
+ *
+ * @param name property name
+ * @return property value interpreted as an array of comma-delimited
+ * <code>int</code> values
+ */
+ public int[] getInts(String name) {
+ String[] strings = getTrimmedStrings(name);
+ int[] ints = new int[strings.length];
+ for (int i = 0; i < strings.length; i++) {
+ ints[i] = Integer.parseInt(strings[i]);
+ }
+ return ints;
+ }
+
+ /**
+ * Set the value of the <code>name</code> property to an <code>int</code>.
+ *
+ * @param name property name.
+ * @param value <code>int</code> value of the property.
+ */
+ public void setInt(String name, int value) {
+ set(name, Integer.toString(value));
+ }
+
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>long</code>.
+ * If no such property exists, the provided default value is returned,
+ * or if the specified value is not a valid <code>long</code>,
+ * then an error is thrown.
+ *
+ * @param name property name.
+ * @param defaultValue default value.
+ * @throws NumberFormatException when the value is invalid
+ * @return property value as a <code>long</code>,
+ * or <code>defaultValue</code>.
+ */
+ public long getLong(String name, long defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null)
+ return defaultValue;
+ String hexString = getHexDigits(valueString);
+ if (hexString != null) {
+ return Long.parseLong(hexString, 16);
+ }
+ return Long.parseLong(valueString);
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>long</code> or
+ * human readable format. If no such property exists, the provided default
+ * value is returned, or if the specified value is not a valid
+ * <code>long</code> or human readable format, then an error is thrown. You
+ * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ * t(tera), p(peta), e(exa)
+ *
+ * @param name property name.
+ * @param defaultValue default value.
+ * @throws NumberFormatException when the value is invalid
+ * @return property value as a <code>long</code>,
+ * or <code>defaultValue</code>.
+ */
+ public long getLongBytes(String name, long defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null)
+ return defaultValue;
+ return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
+ }
+
+ private String getHexDigits(String value) {
+ boolean negative = false;
+ String str = value;
+ String hexString = null;
+ if (value.startsWith("-")) {
+ negative = true;
+ str = value.substring(1);
+ }
+ if (str.startsWith("0x") || str.startsWith("0X")) {
+ hexString = str.substring(2);
+ if (negative) {
+ hexString = "-" + hexString;
+ }
+ return hexString;
+ }
+ return null;
+ }
+
+ /**
+ * Set the value of the <code>name</code> property to a <code>long</code>.
+ *
+ * @param name property name.
+ * @param value <code>long</code> value of the property.
+ */
+ public void setLong(String name, long value) {
+ set(name, Long.toString(value));
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>float</code>.
+ * If no such property exists, the provided default value is returned,
+ * or if the specified value is not a valid <code>float</code>,
+ * then an error is thrown.
+ *
+ * @param name property name.
+ * @param defaultValue default value.
+ * @throws NumberFormatException when the value is invalid
+ * @return property value as a <code>float</code>,
+ * or <code>defaultValue</code>.
+ */
+ public float getFloat(String name, float defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null)
+ return defaultValue;
+ return Float.parseFloat(valueString);
+ }
+
+ /**
+ * Set the value of the <code>name</code> property to a <code>float</code>.
+ *
+ * @param name property name.
+ * @param value property value.
+ */
+ public void setFloat(String name, float value) {
+ set(name,Float.toString(value));
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>double</code>.
+ * If no such property exists, the provided default value is returned,
+ * or if the specified value is not a valid <code>double</code>,
+ * then an error is thrown.
+ *
+ * @param name property name.
+ * @param defaultValue default value.
+ * @throws NumberFormatException when the value is invalid
+ * @return property value as a <code>double</code>,
+ * or <code>defaultValue</code>.
+ */
+ public double getDouble(String name, double defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null)
+ return defaultValue;
+ return Double.parseDouble(valueString);
+ }
+
+ /**
+ * Set the value of the <code>name</code> property to a <code>double</code>.
+ *
+ * @param name property name.
+ * @param value property value.
+ */
+ public void setDouble(String name, double value) {
+ set(name,Double.toString(value));
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>boolean</code>.
+ * If no such property is specified, or if the specified value is not a valid
+ * <code>boolean</code>, then <code>defaultValue</code> is returned.
+ *
+ * @param name property name.
+ * @param defaultValue default value.
+ * @return property value as a <code>boolean</code>,
+ * or <code>defaultValue</code>.
+ */
+ public boolean getBoolean(String name, boolean defaultValue) {
+ String valueString = getTrimmed(name);
+ if (null == valueString || valueString.isEmpty()) {
+ return defaultValue;
+ }
+
+ if (StringUtils.equalsIgnoreCase("true", valueString))
+ return true;
+ else if (StringUtils.equalsIgnoreCase("false", valueString))
+ return false;
+ else return defaultValue;
+ }
+
+ /**
+ * Set the value of the <code>name</code> property to a <code>boolean</code>.
+ *
+ * @param name property name.
+ * @param value <code>boolean</code> value of the property.
+ */
+ public void setBoolean(String name, boolean value) {
+ set(name, Boolean.toString(value));
+ }
+
+ /**
+ * Set the given property, if it is currently unset.
+ * @param name property name
+ * @param value new value
+ */
+ public void setBooleanIfUnset(String name, boolean value) {
+ setIfUnset(name, Boolean.toString(value));
+ }
+
+ /**
+ * Set the value of the <code>name</code> property to the given type. This
+ * is equivalent to <code>set(<name>, value.toString())</code>.
+ * @param name property name
+ * @param value new value
+ */
+ public <T extends Enum<T>> void setEnum(String name, T value) {
+ set(name, value.toString());
+ }
+
+ /**
+ * Return value matching this enumerated type.
+ * Note that the returned value is trimmed by this method.
+ * @param name Property name
+ * @param defaultValue Value returned if no mapping exists
+ * @throws IllegalArgumentException If mapping is illegal for the type
+ * provided
+ */
+ public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
+ final String val = getTrimmed(name);
+ return null == val
+ ? defaultValue
+ : Enum.valueOf(defaultValue.getDeclaringClass(), val);
+ }
+
+ enum ParsedTimeDuration {
+ NS {
+ TimeUnit unit() { return TimeUnit.NANOSECONDS; }
+ String suffix() { return "ns"; }
+ },
+ US {
+ TimeUnit unit() { return TimeUnit.MICROSECONDS; }
+ String suffix() { return "us"; }
+ },
+ MS {
+ TimeUnit unit() { return TimeUnit.MILLISECONDS; }
+ String suffix() { return "ms"; }
+ },
+ S {
+ TimeUnit unit() { return TimeUnit.SECONDS; }
+ String suffix() { return "s"; }
+ },
+ M {
+ TimeUnit unit() { return TimeUnit.MINUTES; }
+ String suffix() { return "m"; }
+ },
+ H {
+ TimeUnit unit() { return TimeUnit.HOURS; }
+ String suffix() { return "h"; }
+ },
+ D {
+ TimeUnit unit() { return TimeUnit.DAYS; }
+ String suffix() { return "d"; }
+ };
+ abstract TimeUnit unit();
+ abstract String suffix();
+ static ParsedTimeDuration unitFor(String s) {
+ for (ParsedTimeDuration ptd : values()) {
+ // iteration order is in decl order, so SECONDS matched last
+ if (s.endsWith(ptd.suffix())) {
+ return ptd;
+ }
+ }
+ return null;
+ }
+ static ParsedTimeDuration unitFor(TimeUnit unit) {
+ for (ParsedTimeDuration ptd : values()) {
+ if (ptd.unit() == unit) {
+ return ptd;
+ }
+ }
+ return null;
+ }
+ }
+
+ /**
+ * Set the value of <code>name</code> to the given time duration. This
+ * is equivalent to <code>set(<name>, value + <time suffix>)</code>.
+ * @param name Property name
+ * @param value Time duration
+ * @param unit Unit of time
+ */
+ public void setTimeDuration(String name, long value, TimeUnit unit) {
+ set(name, value + ParsedTimeDuration.unitFor(unit).suffix());
+ }
+
+ /**
+ * Return time duration in the given time unit. Valid units are encoded in
+ * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ * (ms), seconds (s), minutes (m), hours (h), and days (d).
+ * @param name Property name
+ * @param defaultValue Value returned if no mapping exists.
+ * @param unit Unit to convert the stored property, if it exists.
+ * @throws NumberFormatException If the property stripped of its unit is not
+ * a number
+ */
+ public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
+ String vStr = get(name);
+ if (null == vStr) {
+ return defaultValue;
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
+ }
+ }
+
+ public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
+ String vStr = get(name);
+ if (null == vStr) {
+ return getTimeDurationHelper(name, defaultValue, unit);
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
+ }
+ }
+
+ /**
+ * Return time duration in the given time unit. Valid units are encoded in
+ * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ * (ms), seconds (s), minutes (m), hours (h), and days (d).
+ * @param name Property name
+ * @param vStr The string value with time unit suffix to be converted.
+ * @param unit Unit to convert the stored property, if it exists.
+ */
+ public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
+ vStr = vStr.trim();
+ vStr = StringUtils.toLowerCase(vStr);
+ ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
+ if (null == vUnit) {
+ logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + unit);
+ vUnit = ParsedTimeDuration.unitFor(unit);
+ } else {
+ vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
+ }
+
+ long raw = Long.parseLong(vStr);
+ long converted = unit.convert(raw, vUnit.unit());
+ if (vUnit.unit().convert(converted, unit) < raw) {
+ logDeprecation("Possible loss of precision converting " + vStr
+ + vUnit.suffix() + " to " + unit + " for " + name);
+ }
+ return converted;
+ }
+
+ public long[] getTimeDurations(String name, TimeUnit unit) {
+ String[] strings = getTrimmedStrings(name);
+ long[] durations = new long[strings.length];
+ for (int i = 0; i < strings.length; i++) {
+ durations[i] = getTimeDurationHelper(name, strings[i], unit);
+ }
+ return durations;
+ }
+ /**
+ * Gets the Storage Size from the config, or returns the defaultValue. The
+ * unit of return value is specified in target unit.
+ *
+ * @param name - Key Name
+ * @param defaultValue - Default Value -- e.g. 100MB
+ * @param targetUnit - The units that we want result to be in.
+ * @return double -- formatted in target Units
+ */
+ public double getStorageSize(String name, String defaultValue,
+ StorageUnit targetUnit) {
+ Preconditions.checkState(isNotBlank(name), "Key cannot be blank.");
+ String vString = get(name);
+ if (isBlank(vString)) {
+ vString = defaultValue;
+ }
+
+ // Please note: There is a bit of subtlety here. If the user specifies
+ // the default unit as "1GB", but the requested unit is MB, we will return
+ // the format in MB even thought the default string is specified in GB.
+
+ // Converts a string like "1GB" to to unit specified in targetUnit.
+
+ StorageSize measure = StorageSize.parse(vString);
+ return convertStorageUnit(measure.getValue(), measure.getUnit(),
+ targetUnit);
+ }
+
+ /**
+ * Gets storage size from a config file.
+ *
+ * @param name - Key to read.
+ * @param defaultValue - The default value to return in case the key is
+ * not present.
+ * @param targetUnit - The Storage unit that should be used
+ * for the return value.
+ * @return - double value in the Storage Unit specified.
+ */
+ public double getStorageSize(String name, double defaultValue,
+ StorageUnit targetUnit) {
+ Preconditions.checkNotNull(targetUnit, "Conversion unit cannot be null.");
+ Preconditions.checkState(isNotBlank(name), "Name cannot be blank.");
+ String vString = get(name);
+ if (isBlank(vString)) {
+ return targetUnit.getDefault(defaultValue);
+ }
+
+ StorageSize measure = StorageSize.parse(vString);
+ return convertStorageUnit(measure.getValue(), measure.getUnit(),
+ targetUnit);
+
+ }
+
+ /**
+ * Sets Storage Size for the specified key.
+ *
+ * @param name - Key to set.
+ * @param value - The numeric value to set.
+ * @param unit - Storage Unit to be used.
+ */
+ public void setStorageSize(String name, double value, StorageUnit unit) {
+ set(name, value + unit.getShortName());
+ }
+
+ /**
+ * convert the value from one storage unit to another.
+ *
+ * @param value - value
+ * @param sourceUnit - Source unit to convert from
+ * @param targetUnit - target unit.
+ * @return double.
+ */
+ private double convertStorageUnit(double value, StorageUnit sourceUnit,
+ StorageUnit targetUnit) {
+ double byteValue = sourceUnit.toBytes(value);
+ return targetUnit.fromBytes(byteValue);
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ * If no such property is specified, or if the specified value is not a valid
+ * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+ * Note that the returned value is NOT trimmed by this method.
+ *
+ * @param name property name
+ * @param defaultValue default value
+ * @return property value as a compiled Pattern, or defaultValue
+ */
+ public Pattern getPattern(String name, Pattern defaultValue) {
+ String valString = get(name);
+ if (null == valString || valString.isEmpty()) {
+ return defaultValue;
+ }
+ try {
+ return Pattern.compile(valString);
+ } catch (PatternSyntaxException pse) {
+ LOG.warn("Regular expression '" + valString + "' for property '" +
+ name + "' not valid. Using default", pse);
+ return defaultValue;
+ }
+ }
+
+ /**
+ * Set the given property to <code>Pattern</code>.
+ * If the pattern is passed as null, sets the empty pattern which results in
+ * further calls to getPattern(...) returning the default value.
+ *
+ * @param name property name
+ * @param pattern new value
+ */
+ public void setPattern(String name, Pattern pattern) {
+ assert pattern != null : "Pattern cannot be null";
+ set(name, pattern.pattern());
+ }
+
+ /**
+ * Gets information about why a property was set. Typically this is the
+ * path to the resource objects (file, URL, etc.) the property came from, but
+ * it can also indicate that it was set programmatically, or because of the
+ * command line.
+ *
+ * @param name - The property name to get the source of.
+ * @return null - If the property or its source wasn't found. Otherwise,
+ * returns a list of the sources of the resource. The older sources are
+ * the first ones in the list. So for example if a configuration is set from
+ * the command line, and then written out to a file that is read back in the
+ * first entry would indicate that it was set from the command line, while
+ * the second one would indicate the file that the new configuration was read
+ * in from.
+ */
+ @InterfaceStability.Unstable
+ public synchronized String[] getPropertySources(String name) {
+ if (properties == null) {
+ // If properties is null, it means a resource was newly added
+ // but the props were cleared so as to load it upon future
+ // requests. So lets force a load by asking a properties list.
+ getProps();
+ }
+ // Return a null right away if our properties still
+ // haven't loaded or the resource mapping isn't defined
+ if (properties == null || updatingResource == null) {
+ return null;
+ } else {
+ String[] source = updatingResource.get(name);
+ if(source == null) {
+ return null;
+ } else {
+ return Arrays.copyOf(source, source.length);
+ }
+ }
+ }
+
+ /**
+ * A class that represents a set of positive integer ranges. It parses
+ * strings of the form: "2-3,5,7-" where ranges are separated by comma and
+ * the lower/upper bounds are separated by dash. Either the lower or upper
+ * bound may be omitted meaning all values up to or over. So the string
+ * above means 2, 3, 5, and 7, 8, 9, ...
+ */
+ public static class IntegerRanges implements Iterable<Integer>{
+ private static class Range {
+ int start;
+ int end;
+ }
+
+ private static class RangeNumberIterator implements Iterator<Integer> {
+ Iterator<Range> internal;
+ int at;
+ int end;
+
+ public RangeNumberIterator(List<Range> ranges) {
+ if (ranges != null) {
+ internal = ranges.iterator();
+ }
+ at = -1;
+ end = -2;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (at <= end) {
+ return true;
+ } else if (internal != null){
+ return internal.hasNext();
+ }
+ return false;
+ }
+
+ @Override
+ public Integer next() {
+ if (at <= end) {
+ at++;
+ return at - 1;
+ } else if (internal != null){
+ Range found = internal.next();
+ if (found != null) {
+ at = found.start;
+ end = found.end;
+ at++;
+ return at - 1;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ List<Range> ranges = new ArrayList<Range>();
+
+ public IntegerRanges() {
+ }
+
+ public IntegerRanges(String newValue) {
+ StringTokenizer itr = new StringTokenizer(newValue, ",");
+ while (itr.hasMoreTokens()) {
+ String rng = itr.nextToken().trim();
+ String[] parts = rng.split("-", 3);
+ if (parts.length < 1 || parts.length > 2) {
+ throw new IllegalArgumentException("integer range badly formed: " +
+ rng);
+ }
+ Range r = new Range();
+ r.start = convertToInt(parts[0], 0);
+ if (parts.length == 2) {
+ r.end = convertToInt(parts[1], Integer.MAX_VALUE);
+ } else {
+ r.end = r.start;
+ }
+ if (r.start > r.end) {
+ throw new IllegalArgumentException("IntegerRange from " + r.start +
+ " to " + r.end + " is invalid");
+ }
+ ranges.add(r);
+ }
+ }
+
+ /**
+ * Convert a string to an int treating empty strings as the default value.
+ * @param value the string value
+ * @param defaultValue the value for if the string is empty
+ * @return the desired integer
+ */
+ private static int convertToInt(String value, int defaultValue) {
+ String trim = value.trim();
+ if (trim.length() == 0) {
+ return defaultValue;
+ }
+ return Integer.parseInt(trim);
+ }
+
+ /**
+ * Is the given value in the set of ranges
+ * @param value the value to check
+ * @return is the value in the ranges?
+ */
+ public boolean isIncluded(int value) {
+ for(Range r: ranges) {
+ if (r.start <= value && value <= r.end) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * @return true if there are no values in this range, else false.
+ */
+ public boolean isEmpty() {
+ return ranges == null || ranges.isEmpty();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ boolean first = true;
+ for(Range r: ranges) {
+ if (first) {
+ first = false;
+ } else {
+ result.append(',');
+ }
+ result.append(r.start);
+ result.append('-');
+ result.append(r.end);
+ }
+ return result.toString();
+ }
+
+ /**
+ * Get range start for the first integer range.
+ * @return range start.
+ */
+ public int getRangeStart() {
+ if (ranges == null || ranges.isEmpty()) {
+ return -1;
+ }
+ Range r = ranges.get(0);
+ return r.start;
+ }
+
+ @Override
+ public Iterator<Integer> iterator() {
+ return new RangeNumberIterator(ranges);
+ }
+
+ }
+
+ /**
+ * Parse the given attribute as a set of integer ranges
+ * @param name the attribute name
+ * @param defaultValue the default value if it is not set
+ * @return a new set of ranges from the configured value
+ */
+ public IntegerRanges getRange(String name, String defaultValue) {
+ return new IntegerRanges(get(name, defaultValue));
+ }
+
+ /**
+ * Get the comma delimited values of the <code>name</code> property as
+ * a collection of <code>String</code>s.
+ * If no such property is specified then empty collection is returned.
+ * <p>
+ * This is an optimized version of {@link #getStrings(String)}
+ *
+ * @param name property name.
+ * @return property value as a collection of <code>String</code>s.
+ */
+ public Collection<String> getStringCollection(String name) {
+ String valueString = get(name);
+ return StringUtils.getStringCollection(valueString);
+ }
+
+ /**
+ * Get the comma delimited values of the <code>name</code> property as
+ * an array of <code>String</code>s.
+ * If no such property is specified then <code>null</code> is returned.
+ *
+ * @param name property name.
+ * @return property value as an array of <code>String</code>s,
+ * or <code>null</code>.
+ */
+ public String[] getStrings(String name) {
+ String valueString = get(name);
+ return StringUtils.getStrings(valueString);
+ }
+
+ /**
+ * Get the comma delimited values of the <code>name</code> property as
+ * an array of <code>String</code>s.
+ * If no such property is specified then default value is returned.
+ *
+ * @param name property name.
+ * @param defaultValue The default value
+ * @return property value as an array of <code>String</code>s,
+ * or default value.
+ */
+ public String[] getStrings(String name, String... defaultValue) {
+ String valueString = get(name);
+ if (valueString == null) {
+ return defaultValue;
+ } else {
+ return StringUtils.getStrings(valueString);
+ }
+ }
+
+ /**
+ * Get the comma delimited values of the <code>name</code> property as
+ * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ * If no such property is specified then empty <code>Collection</code> is returned.
+ *
+ * @param name property name.
+ * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>
+ */
+ public Collection<String> getTrimmedStringCollection(String name) {
+ String valueString = get(name);
+ if (null == valueString) {
+ Collection<String> empty = new ArrayList<String>();
+ return empty;
+ }
+ return StringUtils.getTrimmedStringCollection(valueString);
+ }
+
+ /**
+ * Get the comma delimited values of the <code>name</code> property as
+ * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ * If no such property is specified then an empty array is returned.
+ *
+ * @param name property name.
+ * @return property value as an array of trimmed <code>String</code>s,
+ * or empty array.
+ */
+ public String[] getTrimmedStrings(String name) {
+ String valueString = get(name);
+ return StringUtils.getTrimmedStrings(valueString);
+ }
+
+ /**
+ * Get the comma delimited values of the <code>name</code> property as
+ * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ * If no such property is specified then default value is returned.
+ *
+ * @param name property name.
+ * @param defaultValue The default value
+ * @return property value as an array of trimmed <code>String</code>s,
+ * or default value.
+ */
+ public String[] getTrimmedStrings(String name, String... defaultValue) {
+ String valueString = get(name);
+ if (null == valueString) {
+ return defaultValue;
+ } else {
+ return StringUtils.getTrimmedStrings(valueString);
+ }
+ }
+
+ /**
+ * Set the array of string values for the <code>name</code> property as
+ * as comma delimited values.
+ *
+ * @param name property name.
+ * @param values The values
+ */
+ public void setStrings(String name, String... values) {
+ set(name, StringUtils.arrayToString(values));
+ }
+
+ /**
+ * Get the value for a known password configuration element.
+ * In order to enable the elimination of clear text passwords in config,
+ * this method attempts to resolve the property name as an alias through
+ * the CredentialProvider API and conditionally fallsback to config.
+ * @param name property name
+ * @return password
+ */
+ public char[] getPassword(String name) throws IOException {
+ char[] pass = null;
+
+ pass = getPasswordFromCredentialProviders(name);
+
+ if (pass == null) {
+ pass = getPasswordFromConfig(name);
+ }
+
+ return pass;
+ }
+
+ /**
+ * Get the credential entry by name from a credential provider.
+ *
+ * Handle key deprecation.
+ *
+ * @param provider a credential provider
+ * @param name alias of the credential
+ * @return the credential entry or null if not found
+ */
+ private CredentialEntry getCredentialEntry(CredentialProvider provider,
+ String name) throws IOException {
+ CredentialEntry entry = provider.getCredentialEntry(name);
+ if (entry != null) {
+ return entry;
+ }
+
+ // The old name is stored in the credential provider.
+ String oldName = getDeprecatedKey(name);
+ if (oldName != null) {
+ entry = provider.getCredentialEntry(oldName);
+ if (entry != null) {
+ logDeprecationOnce(oldName, provider.toString());
+ return entry;
+ }
+ }
+
+ // The name is deprecated.
+ DeprecatedKeyInfo keyInfo = getDeprecatedKeyInfo(name);
+ if (keyInfo != null && keyInfo.newKeys != null) {
+ for (String newName : keyInfo.newKeys) {
+ entry = provider.getCredentialEntry(newName);
+ if (entry != null) {
+ logDeprecationOnce(name, null);
+ return entry;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Try and resolve the provided element name as a credential provider
+ * alias.
+ * @param name alias of the provisioned credential
+ * @return password or null if not found
+ * @throws IOException
+ */
+ protected char[] getPasswordFromCredentialProviders(String name)
+ throws IOException {
+ char[] pass = null;
+ try {
+ List<CredentialProvider> providers =
+ CredentialProviderFactory.getProviders(this);
+
+ if (providers != null) {
+ for (CredentialProvider provider : providers) {
+ try {
+ CredentialEntry entry = getCredentialEntry(provider, name);
+ if (entry != null) {
+ pass = entry.getCredential();
+ break;
+ }
+ }
+ catch (IOException ioe) {
+ throw new IOException("Can't get key " + name + " from key provider" +
+ "of type: " + provider.getClass().getName() + ".", ioe);
+ }
+ }
+ }
+ }
+ catch (IOException ioe) {
+ throw new IOException("Configuration problem with provider path.", ioe);
+ }
+
+ return pass;
+ }
+
+ /**
+ * Fallback to clear text passwords in configuration.
+ * @param name
+ * @return clear text password or null
+ */
+ protected char[] getPasswordFromConfig(String name) {
+ char[] pass = null;
+ if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK,
+ CommonConfigurationKeysPublic.
+ HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT)) {
+ String passStr = get(name);
+ if (passStr != null) {
+ pass = passStr.toCharArray();
+ }
+ }
+ return pass;
+ }
+
+ /**
+ * Get the socket address for <code>hostProperty</code> as a
+ * <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ * <code>null</code>, <code>addressProperty</code> will be used. This
+ * is useful for cases where we want to differentiate between host
+ * bind address and address clients should use to establish connection.
+ *
+ * @param hostProperty bind host property name.
+ * @param addressProperty address property name.
+ * @param defaultAddressValue the default value
+ * @param defaultPort the default port
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getSocketAddr(
+ String hostProperty,
+ String addressProperty,
+ String defaultAddressValue,
+ int defaultPort) {
+
+ InetSocketAddress bindAddr = getSocketAddr(
+ addressProperty, defaultAddressValue, defaultPort);
+
+ final String host = get(hostProperty);
+
+ if (host == null || host.isEmpty()) {
+ return bindAddr;
+ }
+
+ return NetUtils.createSocketAddr(
+ host, bindAddr.getPort(), hostProperty);
+ }
+
+ /**
+ * Get the socket address for <code>name</code> property as a
+ * <code>InetSocketAddress</code>.
+ * @param name property name.
+ * @param defaultAddress the default value
+ * @param defaultPort the default port
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getSocketAddr(
+ String name, String defaultAddress, int defaultPort) {
+ final String address = getTrimmed(name, defaultAddress);
+ return NetUtils.createSocketAddr(address, defaultPort, name);
+ }
+
+ /**
+ * Set the socket address for the <code>name</code> property as
+ * a <code>host:port</code>.
+ */
+ public void setSocketAddr(String name, InetSocketAddress addr) {
+ set(name, NetUtils.getHostPortString(addr));
+ }
+
+ /**
+ * Set the socket address a client can use to connect for the
+ * <code>name</code> property as a <code>host:port</code>. The wildcard
+ * address is replaced with the local host's address. If the host and address
+ * properties are configured the host component of the address will be combined
+ * with the port component of the addr to generate the address. This is to allow
+ * optional control over which host name is used in multi-home bind-host
+ * cases where a host can have multiple names
+ * @param hostProperty the bind-host configuration name
+ * @param addressProperty the service address configuration name
+ * @param defaultAddressValue the service default address configuration value
+ * @param addr InetSocketAddress of the service listener
+ * @return InetSocketAddress for clients to connect
+ */
+ public InetSocketAddress updateConnectAddr(
+ String hostProperty,
+ String addressProperty,
+ String defaultAddressValue,
+ InetSocketAddress addr) {
+
+ final String host = get(hostProperty);
+ final String connectHostPort = getTrimmed(addressProperty, defaultAddressValue);
+
+ if (host == null || host.isEmpty() || connectHostPort == null || connectHostPort.isEmpty()) {
+ //not our case, fall back to original logic
+ return updateConnectAddr(addressProperty, addr);
+ }
+
+ final String connectHost = connectHostPort.split(":")[0];
+ // Create connect address using client address hostname and server port.
+ return updateConnectAddr(addressProperty, NetUtils.createSocketAddrForHost(
+ connectHost, addr.getPort()));
+ }
+
+ /**
+ * Set the socket address a client can use to connect for the
+ * <code>name</code> property as a <code>host:port</code>. The wildcard
+ * address is replaced with the local host's address.
+ * @param name property name.
+ * @param addr InetSocketAddress of a listener to store in the given property
+ * @return InetSocketAddress for clients to connect
+ */
+ public InetSocketAddress updateConnectAddr(String name,
+ InetSocketAddress addr) {
+ final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
+ setSocketAddr(name, connectAddr);
+ return connectAddr;
+ }
+
+ /**
+ * Load a class by name.
+ *
+ * @param name the class name.
+ * @return the class object.
+ * @throws ClassNotFoundException if the class is not found.
+ */
+ public Class<?> getClassByName(String name) throws ClassNotFoundException {
+ Class<?> ret = getClassByNameOrNull(name);
+ if (ret == null) {
+ throw new ClassNotFoundException("Class " + name + " not found");
+ }
+ return ret;
+ }
+
+ /**
+ * Load a class by name, returning null rather than throwing an exception
+ * if it couldn't be loaded. This is to avoid the overhead of creating
+ * an exception.
+ *
+ * @param name the class name
+ * @return the class object, or null if it could not be found.
+ */
+ public Class<?> getClassByNameOrNull(String name) {
+ Map<String, WeakReference<Class<?>>> map;
+
+ synchronized (CACHE_CLASSES) {
+ map = CACHE_CLASSES.get(classLoader);
+ if (map == null) {
+ map = Collections.synchronizedMap(
+ new WeakHashMap<String, WeakReference<Class<?>>>());
+ CACHE_CLASSES.put(classLoader, map);
+ }
+ }
+
+ Class<?> clazz = null;
+ WeakReference<Class<?>> ref = map.get(name);
+ if (ref != null) {
+ clazz = ref.get();
+ }
+
+ if (clazz == null) {
+ try {
+ clazz = Class.forName(name, true, classLoader);
+ } catch (ClassNotFoundException e) {
+ // Leave a marker that the class isn't found
+ map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL));
+ return null;
+ }
+ // two putters can race here, but they'll put the same class
+ map.put(name, new WeakReference<Class<?>>(clazz));
+ return clazz;
+ } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
+ return null; // not found
+ } else {
+ // cache hit
+ return clazz;
+ }
+ }
+
+ /**
+ * Get the value of the <code>name</code> property
+ * as an array of <code>Class</code>.
+ * The value of the property specifies a list of comma separated class names.
+ * If no such property is specified, then <code>defaultValue</code> is
+ * returned.
+ *
+ * @param name the property name.
+ * @param defaultValue default value.
+ * @return property value as a <code>Class[]</code>,
+ * or <code>defaultValue</code>.
+ */
+ public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
+ String valueString = getRaw(name);
+ if (null == valueString) {
+ return defaultValue;
+ }
+ String[] classnames = getTrimmedStrings(name);
+ try {
+ Class<?>[] classes = new Class<?>[classnames.length];
+ for(int i = 0; i < classnames.length; i++) {
+ classes[i] = getClassByName(classnames[i]);
+ }
+ return classes;
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>Class</code>.
+ * If no such property is specified, then <code>defaultValue</code> is
+ * returned.
+ *
+ * @param name the class name.
+ * @param defaultValue default value.
+ * @return property value as a <code>Class</code>,
+ * or <code>defaultValue</code>.
+ */
+ public Class<?> getClass(String name, Class<?> defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null)
+ return defaultValue;
+ try {
+ return getClassByName(valueString);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>Class</code>
+ * implementing the interface specified by <code>xface</code>.
+ *
+ * If no such property is specified, then <code>defaultValue</code> is
+ * returned.
+ *
+ * An exception is thrown if the returned class does not implement the named
+ * interface.
+ *
+ * @param name the class name.
+ * @param defaultValue default value.
+ * @param xface the interface implemented by the named class.
+ * @return property value as a <code>Class</code>,
+ * or <code>defaultValue</code>.
+ */
+ public <U> Class<? extends U> getClass(String name,
+ Class<? extends U> defaultValue,
+ Class<U> xface) {
+ try {
+ Class<?> theClass = getClass(name, defaultValue);
+ if (theClass != null && !xface.isAssignableFrom(theClass))
+ throw new RuntimeException(theClass+" not "+xface.getName());
+ else if (theClass != null)
+ return theClass.asSubclass(xface);
+ else
+ return null;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Get the value of the <code>name</code> property as a <code>List</code>
+ * of objects implementing the interface specified by <code>xface</code>.
+ *
+ * An exception is thrown if any of the classes does not exist, or if it does
+ * not implement the named interface.
+ *
+ * @param name the property name.
+ * @param xface the interface implemented by the classes named by
+ * <code>name</code>.
+ * @return a <code>List</code> of objects implementing <code>xface</code>.
+ */
+ @SuppressWarnings("unchecked")
+ public <U> List<U> getInstances(String name, Class<U> xface) {
+ List<U> ret = new ArrayList<U>();
+ Class<?>[] classes = getClasses(name);
+ for (Class<?> cl: classes) {
+ if (!xface.isAssignableFrom(cl)) {
+ throw new RuntimeException(cl + " does not implement " + xface);
+ }
+ ret.add((U)ReflectionUtils.newInstance(cl, this));
+ }
+ return ret;
+ }
+
+ /**
+ * Set the value of the <code>name</code> property to the name of a
+ * <code>theClass</code> implementing the given interface <code>xface</code>.
+ *
+ * An exception is thrown if <code>theClass</code> does not implement the
+ * interface <code>xface</code>.
+ *
+ * @param name property name.
+ * @param theClass property value.
+ * @param xface the interface implemented by the named class.
+ */
+ public void setClass(String name, Class<?> theClass, Class<?> xface) {
+ if (!xface.isAssignableFrom(theClass))
+ throw new RuntimeException(theClass+" not "+xface.getName());
+ set(name, theClass.getName());
+ }
+
+ /**
+ * Get a local file under a directory named by <i>dirsProp</i> with
+ * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ * then one is chosen based on <i>path</i>'s hash code. If the selected
+ * directory does not exist, an attempt is made to create it.
+ *
+ * @param dirsProp directory in which to locate the file.
+ * @param path file-path.
+ * @return local file under the directory with the given path.
+ */
+ public Path getLocalPath(String dirsProp, String path)
+ throws IOException {
+ String[] dirs = getTrimmedStrings(dirsProp);
+ int hashCode = path.hashCode();
+ FileSystem fs = FileSystem.getLocal(this);
+ for (int i = 0; i < dirs.length; i++) { // try each local dir
+ int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
+ Path file = new Path(dirs[index], path);
+ Path dir = file.getParent();
+ if (fs.mkdirs(dir) || fs.exists(dir)) {
+ return file;
+ }
+ }
+ LOG.warn("Could not make " + path +
+ " in local directories from " + dirsProp);
+ for(int i=0; i < dirs.length; i++) {
+ int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
+ LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
+ }
+ throw new IOException("No valid local directories in property: "+dirsProp);
+ }
+
+ /**
+ * Get a local file name under a directory named in <i>dirsProp</i> with
+ * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ * then one is chosen based on <i>path</i>'s hash code. If the selected
+ * directory does not exist, an attempt is made to create it.
+ *
+ * @param dirsProp directory in which to locate the file.
+ * @param path file-path.
+ * @return local file under the directory with the given path.
+ */
+ public File getFile(String dirsProp, String path)
+ throws IOException {
+ String[] dirs = getTrimmedStrings(dirsProp);
+ int hashCode = path.hashCode();
+ for (int i = 0; i < dirs.length; i++) { // try each local dir
+ int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
+ File file = new File(dirs[index], path);
+ File dir = file.getParentFile();
+ if (dir.exists() || dir.mkdirs()) {
+ return file;
+ }
+ }
+ throw new IOException("No valid local directories in property: "+dirsProp);
+ }
+
+ /**
+ * Get the {@link URL} for the named resource.
+ *
+ * @param name resource name.
+ * @return the url for the named resource.
+ */
+ public URL getResource(String name) {
+ return classLoader.getResource(name);
+ }
+
+ /**
+ * Get an input stream attached to the configuration resource with the
+ * given <code>name</code>.
+ *
+ * @param name configuration resource name.
+ * @return an input stream attached to the resource.
+ */
+ public InputStream getConfResourceAsInputStream(String name) {
+ try {
+ URL url= getResource(name);
+
+ if (url == null) {
+ LOG.info(name + " not found");
+ return null;
+ } else {
+ LOG.info("found resource " + name + " at " + url);
+ }
+
+ return url.openStream();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Get a {@link Reader} attached to the configuration resource with the
+ * given <code>name</code>.
+ *
+ * @param name configuration resource name.
+ * @return a reader attached to the resource.
+ */
+ public Reader getConfResourceAsReader(String name) {
+ try {
+ URL url= getResource(name);
+
+ if (url == null) {
+ LOG.info(name + " not found");
+ return null;
+ } else {
+ LOG.info("found resource " + name + " at " + url);
+ }
+
+ return new InputStreamReader(url.openStream(), Charsets.UTF_8);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * Get the set of parameters marked final.
+ *
+ * @return final parameter set.
+ */
+ public Set<String> getFinalParameters() {
+ Set<String> setFinalParams = Collections.newSetFromMap(
+ new ConcurrentHashMap<String, Boolean>());
+ setFinalParams.addAll(finalParameters);
+ return setFinalParams;
+ }
+
+ protected synchronized Properties getProps() {
+ if (properties == null) {
+ properties = new Properties();
+ Map<String, String[]> backup = updatingResource != null ?
+ new ConcurrentHashMap<String, String[]>(updatingResource) : null;
+ loadResources(properties, resources, quietmode);
+
+ if (overlay != null) {
+ properties.putAll(overlay);
+ if (backup != null) {
+ for (Map.Entry<Object, Object> item : overlay.entrySet()) {
+ String key = (String) item.getKey();
+ String[] source = backup.get(key);
+ if (source != null) {
+ updatingResource.put(key, source);
+ }
+ }
+ }
+ }
+ }
+ return properties;
+ }
+
+ /**
+ * Return the number of keys in the configuration.
+ *
+ * @return number of keys in the configuration.
+ */
+ public int size() {
+ return getProps().size();
+ }
+
+ /**
+ * Clears all keys from the configuration.
+ */
+ public void clear() {
+ getProps().clear();
+ getOverlay().clear();
+ }
+
+ /**
+ * Get an {@link Iterator} to go through the list of <code>String</code>
+ * key-value pairs in the configuration.
+ *
+ * @return an iterator over the entries.
+ */
+ @Override
+ public Iterator<Map.Entry<String, String>> iterator() {
+ // Get a copy of just the string to string pairs. After the old object
+ // methods that allow non-strings to be put into configurations are removed,
+ // we could replace properties with a Map<String,String> and get rid of this
+ // code.
+ Map<String,String> result = new HashMap<String,String>();
+ for(Map.Entry<Object,Object> item: getProps().entrySet()) {
+ if (item.getKey() instanceof String &&
+ item.getValue() instanceof String) {
+ result.put((String) item.getKey(), (String) item.getValue());
+ }
+ }
+ return result.entrySet().iterator();
+ }
+
+ /**
+ * Constructs a mapping of configuration and includes all properties that
+ * start with the specified configuration prefix. Property names in the
+ * mapping are trimmed to remove the configuration prefix.
+ *
+ * @param confPrefix configuration prefix
+ * @return mapping of configuration properties with prefix stripped
+ */
+ public Map<String, String> getPropsWithPrefix(String confPrefix) {
+ Properties props = getProps();
+ Enumeration e = props.propertyNames();
+ Map<String, String> configMap = new HashMap<>();
+ String name = null;
+ while (e.hasMoreElements()) {
+ name = (String) e.nextElement();
+ if (name.startsWith(confPrefix)) {
+ String value = props.getProperty(name);
+ name = name.substring(confPrefix.length());
+ configMap.put(name, value);
+ }
+ }
+ return configMap;
+ }
+
+ private XMLStreamReader parse(URL url, boolean restricted)
+ throws IOException, XMLStreamException {
+ if (!quietmode) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("parsing URL " + url);
+ }
+ }
+ if (url == null) {
+ return null;
+ }
+
+ URLConnection connection = url.openConnection();
+ if (connection instanceof JarURLConnection) {
+ // Disable caching for JarURLConnection to avoid sharing JarFile
+ // with other users.
+ connection.setUseCaches(false);
+ }
+ return parse(connection.getInputStream(), url.toString(), restricted);
+ }
+
+ private XMLStreamReader parse(InputStream is, String systemIdStr,
+ boolean restricted) throws IOException, XMLStreamException {
+ if (!quietmode) {
+ LOG.debug("parsing input stream " + is);
+ }
+ if (is == null) {
+ return null;
+ }
+ SystemId systemId = SystemId.construct(systemIdStr);
+ ReaderConfig readerConfig = XML_INPUT_FACTORY.createPrivateConfig();
+ if (restricted) {
+ readerConfig.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ }
+ return XML_INPUT_FACTORY.createSR(readerConfig, systemId,
+ StreamBootstrapper.getInstance(null, systemId, is), false, true);
+ }
+
+ private void loadResources(Properties properties,
+ ArrayList<Resource> resources,
+ boolean quiet) {
+ if(loadDefaults) {
+ for (String resource : defaultResources) {
+ loadResource(properties, new Resource(resource, false), quiet);
+ }
+ }
+
+ for (int i = 0; i < resources.size(); i++) {
+ Resource ret = loadResource(properties, resources.get(i), quiet);
+ if (ret != null) {
+ resources.set(i, ret);
+ }
+ }
+ }
+
+ private Resource loadResource(Properties properties,
+ Resource wrapper, boolean quiet) {
+ String name = UNKNOWN_RESOURCE;
+ try {
+ Object resource = wrapper.getResource();
+ name = wrapper.getName();
+ XMLStreamReader2 reader = null;
+ boolean returnCachedProperties = false;
+ boolean isRestricted = wrapper.isParserRestricted();
+
+ if (resource instanceof URL) { // an URL resource
+ reader = (XMLStreamReader2)parse((URL)resource, isRestricted);
+ } else if (resource instanceof String) { // a CLASSPATH resource
+ URL url = getResource((String)resource);
+ reader = (XMLStreamReader2)parse(url, isRestricted);
+ } else if (resource instanceof Path) { // a file resource
+ // Can't use FileSystem API or we get an infinite loop
+ // since FileSystem uses Configuration API. Use java.io.File instead.
+ File file = new File(((Path)resource).toUri().getPath())
+ .getAbsoluteFile();
+ if (file.exists()) {
+ if (!quiet) {
+ LOG.debug("parsing File " + file);
+ }
+ reader = (XMLStreamReader2)parse(new BufferedInputStream(
+ new FileInputStream(file)), ((Path)resource).toString(),
+ isRestricted);
+ }
+ } else if (resource instanceof InputStream) {
+ reader = (XMLStreamReader2)parse((InputStream)resource, null,
+ isRestricted);
+ returnCachedProperties = true;
+ } else if (resource instanceof Properties) {
+ overlay(properties, (Properties)resource);
+ }
+
+ if (reader == null) {
+ if (quiet) {
+ return null;
+ }
+ throw new RuntimeException(resource + " not found");
+ }
+ Properties toAddTo = properties;
+ if(returnCachedProperties) {
+ toAddTo = new Properties();
+ }
+ DeprecationContext deprecations = deprecationContext.get();
+
+ StringBuilder token = new StringBuilder();
+ String confName = null;
+ String confValue = null;
+ String confInclude = null;
+ boolean confFinal = false;
+ boolean fallbackAllowed = false;
+ boolean fallbackEntered = false;
+ boolean parseToken = false;
+ LinkedList<String> confSource = new LinkedList<String>();
+
+ while (reader.hasNext()) {
+ switch (reader.next()) {
+ case XMLStreamConstants.START_ELEMENT:
+ switch (reader.getLocalName()) {
+ case "property":
+ confName = null;
+ confValue = null;
+ confFinal = false;
+ confSource.clear();
+
+ // First test for short format configuration
+ int attrCount = reader.getAttributeCount();
+ for (int i = 0; i < attrCount; i++) {
+ String propertyAttr = reader.getAttributeLocalName(i);
+ if ("name".equals(propertyAttr)) {
+ confName = StringInterner.weakIntern(
+ reader.getAttributeValue(i));
+ } else if ("value".equals(propertyAttr)) {
+ confValue = StringInterner.weakIntern(
+ reader.getAttributeValue(i));
+ } else if ("final".equals(propertyAttr)) {
+ confFinal = "true".equals(reader.getAttributeValue(i));
+ } else if ("source".equals(propertyAttr)) {
+ confSource.add(StringInterner.weakIntern(
+ reader.getAttributeValue(i)));
+ }
+ }
+ break;
+ case "name":
+ case "value":
+ case "final":
+ case "source":
+ parseToken = true;
+ token.setLength(0);
+ break;
+ case "include":
+ // Determine href for xi:include
+ confInclude = null;
+ attrCount = reader.getAttributeCount();
+ for (int i = 0; i < attrCount; i++) {
+ String attrName = reader.getAttributeLocalName(i);
+ if ("href".equals(attrName)) {
+ confInclude = reader.getAttributeValue(i);
+ }
+ }
+ if (confInclude == null) {
+ break;
+ }
+ if (isRestricted) {
+ throw new RuntimeException("Error parsing resource " + wrapper
+ + ": XInclude is not supported for restricted resources");
+ }
+ // Determine if the included resource is a classpath resource
+ // otherwise fallback to a file resource
+ // xi:include are treated as inline and retain current source
+ URL include = getResource(confInclude);
+ if (include != null) {
+ Resource classpathResource = new Resource(include, name,
+ wrapper.isParserRestricted());
+ loadResource(properties, classpathResource, quiet);
+ } else {
+ URL url;
+ try {
+ url = new URL(confInclude);
+ url.openConnection().connect();
+ } catch (IOException ioe) {
+ File href = new File(confInclude);
+ if (!href.isAbsolute()) {
+ // Included resources are relative to the current resource
+ File baseFile = new File(name).getParentFile();
+ href = new File(baseFile, href.getPath());
+ }
+ if (!href.exists()) {
+ // Resource errors are non-fatal iff there is 1 xi:fallback
+ fallbackAllowed = true;
+ break;
+ }
+ url = href.toURI().toURL();
+ }
+ Resource uriResource = new Resource(url, name,
+ wrapper.isParserRestricted());
+ loadResource(properties, uriResource, quiet);
+ }
+ break;
+ case "fallback":
+ fallbackEntered = true;
+ break;
+ case "configuration":
+ break;
+ default:
+ break;
+ }
+ break;
+
+ case XMLStreamConstants.CHARACTERS:
+ if (parseToken) {
+ char[] text = reader.getTextCharacters();
+ token.append(text, reader.getTextStart(), reader.getTextLength());
+ }
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ switch (reader.getLocalName()) {
+ case "name":
+ if (token.length() > 0) {
+ confName = StringInterner.weakIntern(token.toString().trim());
+ }
+ break;
+ case "value":
+ if (token.length() > 0) {
+ confValue = StringInterner.weakIntern(token.toString());
+ }
+ break;
+ case "final":
+ confFinal = "true".equals(token.toString());
+ break;
+ case "source":
+ confSource.add(StringInterner.weakIntern(token.toString()));
+ break;
+ case "include":
+ if (fallbackAllowed && !fallbackEntered) {
+ throw new IOException("Fetch fail on include for '"
+ + confInclude + "' with no fallback while loading '"
+ + name + "'");
+ }
+ fallbackAllowed = false;
+ fallbackEntered = false;
+ break;
+ case "property":
+ if (confName == null || (!fallbackAllowed && fallbackEntered)) {
+ break;
+ }
+ confSource.add(name);
+ DeprecatedKeyInfo keyInfo =
+ deprecations.getDeprecatedKeyMap().get(confName);
+ if (keyInfo != null) {
+ keyInfo.clearAccessed();
+ for (String key : keyInfo.newKeys) {
+ // update new keys with deprecated key's value
+ loadProperty(toAddTo, name, key, confValue, confFinal,
+ confSource.toArray(new String[confSource.size()]));
+ }
+ } else {
+ loadProperty(toAddTo, name, confName, confValue, confFinal,
+ confSource.toArray(new String[confSource.size()]));
+ }
+ break;
+ default:
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ reader.close();
+
+ if (returnCachedProperties) {
+ overlay(properties, toAddTo);
+ return new Resource(toAddTo, name, wrapper.isParserRestricted());
+ }
+ return null;
+ } catch (IOException e) {
+ LOG.error("error parsing conf " + name, e);
+ throw new RuntimeException(e);
+ } catch (XMLStreamException e) {
+ LOG.error("error parsing conf " + name, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void overlay(Properties to, Properties from) {
+ for (Entry<Object, Object> entry: from.entrySet()) {
+ to.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private void loadProperty(Properties properties, String name, String attr,
+ String value, boolean finalParameter, String[] source) {
+ if (value != null || allowNullValueProperties) {
+ if (value == null) {
+ value = DEFAULT_STRING_CHECK;
+ }
+ if (!finalParameters.contains(attr)) {
+ properties.setProperty(attr, value);
+ if (source != null) {
+ putIntoUpdatingResource(attr, source);
+ }
+ } else {
+ // This is a final parameter so check for overrides.
+ checkForOverride(this.properties, name, attr, value);
+ if (this.properties != properties) {
+ checkForOverride(properties, name, attr, value);
+ }
+ }
+ }
+ if (finalParameter && attr != null) {
+ finalParameters.add(attr);
+ }
+ }
+
+ /**
+ * Print a warning if a property with a given name already exists with a
+ * different value
+ */
+ private void checkForOverride(Properties properties, String name, String attr, String value) {
+ String propertyValue = properties.getProperty(attr);
+ if (propertyValue != null && !propertyValue.equals(value)) {
+ LOG.warn(name + ":an attempt to override final parameter: " + attr
+ + "; Ignoring.");
+ }
+ }
+
+ /**
+ * Write out the non-default properties in this configuration to the given
+ * {@link OutputStream} using UTF-8 encoding.
+ *
+ * @param out the output stream to write to.
+ */
+ public void writeXml(OutputStream out) throws IOException {
+ writeXml(new OutputStreamWriter(out, "UTF-8"));
+ }
+
+ public void writeXml(Writer out) throws IOException {
+ writeXml(null, out);
+ }
+
+ /**
+ * Write out the non-default properties in this configuration to the
+ * given {@link Writer}.
+ *
+ * <li>
+ * When property name is not empty and the property exists in the
+ * configuration, this method writes the property and its attributes
+ * to the {@link Writer}.
+ * </li>
+ * <p>
+ *
+ * <li>
+ * When property name is null or empty, this method writes all the
+ * configuration properties and their attributes to the {@link Writer}.
+ * </li>
+ * <p>
+ *
+ * <li>
+ * When property name is not empty but the property doesn't exist in
+ * the configuration, this method throws an {@link IllegalArgumentException}.
+ * </li>
+ * <p>
+ * @param out the writer to write to.
+ */
+ public void writeXml(String propertyName, Writer out)
+ throws IOException, IllegalArgumentException {
+ Document doc = asXmlDocument(propertyName);
+
+ try {
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(out);
+ TransformerFactory transFactory = TransformerFactory.newInstance();
+ Transformer transformer = transFactory.newTransformer();
+
+ // Important to not hold Configuration log while writing result, since
+ // 'out' may be an HDFS stream which needs to lock this configuration
+ // from another thread.
+ transformer.transform(source, result);
+ } catch (TransformerException te) {
+ throw new IOException(te);
+ }
+ }
+
+ /**
+ * Return the XML DOM corresponding to this Configuration.
+ */
+ private synchronized Document asXmlDocument(String propertyName)
+ throws IOException, IllegalArgumentException {
+ Document doc;
+ try {
+ doc = DocumentBuilderFactory
+ .newInstance()
+ .newDocumentBuilder()
+ .newDocument();
+ } catch (ParserConfigurationException pe) {
+ throw new IOException(pe);
+ }
+
+ Element conf = doc.createElement("configuration");
+ doc.appendChild(conf);
+ conf.appendChild(doc.createTextNode("\n"));
+ handleDeprecation(); //ensure properties is set and deprecation is handled
+
+ if(!Strings.isNullOrEmpty(propertyName)) {
+ if (!properties.containsKey(propertyName)) {
+ // given property not found, illegal argument
+ throw new IllegalArgumentException("Property " +
+ propertyName + " not found");
+ } else {
+ // given property is found, write single property
+ appendXMLProperty(doc, conf, propertyName);
+ conf.appendChild(doc.createTextNode("\n"));
+ }
+ } else {
+ // append all elements
+ for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
+ appendXMLProperty(doc, conf, (String)e.nextElement());
+ conf.appendChild(doc.createTextNode("\n"));
+ }
+ }
+ return doc;
+ }
+
+ /**
+ * Append a property with its attributes to a given {#link Document}
+ * if the property is found in configuration.
+ *
+ * @param doc
+ * @param conf
+ * @param propertyName
+ */
+ private synchronized void appendXMLProperty(Document doc, Element conf,
+ String propertyName) {
+ // skip writing if given property name is empty or null
+ if (!Strings.isNullOrEmpty(propertyName)) {
+ String value = properties.getProperty(propertyName);
+ if (value != null) {
+ Element propNode = doc.createElement("property");
+ conf.appendChild(propNode);
+
+ Element nameNode = doc.createElement("name");
+ nameNode.appendChild(doc.createTextNode(propertyName));
+ propNode.appendChild(nameNode);
+
+ Element valueNode = doc.createElement("value");
+ valueNode.appendChild(doc.createTextNode(
+ properties.getProperty(propertyName)));
+ propNode.appendChild(valueNode);
+
+ Element finalNode = doc.createElement("final");
+ finalNode.appendChild(doc.createTextNode(
+ String.valueOf(finalParameters.contains(propertyName))));
+ propNode.appendChild(finalNode);
+
+ if (updatingResource != null) {
+ String[] sources = updatingResource.get(propertyName);
+ if(sources != null) {
+ for(String s : sources) {
+ Element sourceNode = doc.createElement("source");
+ sourceNode.appendChild(doc.createTextNode(s));
+ propNode.appendChild(sourceNode);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Writes properties and their attributes (final and resource)
+ * to the given {@link Writer}.
+ *
+ * <li>
+ * When propertyName is not empty, and the property exists
+ * in the configuration, the format of the output would be,
+ * <pre>
+ * {
+ * "property": {
+ * "key" : "key1",
+ * "value" : "value1",
+ * "isFinal" : "key1.isFinal",
+ * "resource" : "key1.resource"
+ * }
+ * }
+ * </pre>
+ * </li>
+ *
+ * <li>
+ * When propertyName is null or empty, it behaves same as
+ * {@link #dumpConfiguration(Configuration, Writer)}, the
+ * output would be,
+ * <pre>
+ * { "properties" :
+ * [ { key : "key1",
+ * value : "value1",
+ * isFinal : "key1.isFinal",
+ * resource : "key1.resource" },
+ * { key : "key2",
+ * value : "value2",
+ * isFinal : "ke2.isFinal",
+ * resource : "key2.resource" }
+ * ]
+ * }
+ * </pre>
+ * </li>
+ *
+ * <li>
+ * When propertyName is not empty, and the property is not
+ * found in the configuration, this method will throw an
+ * {@link IllegalArgumentException}.
+ * </li>
+ * <p>
+ * @param config the configuration
+ * @param propertyName property name
+ * @param out the Writer to write to
+ * @throws IOException
+ * @throws IllegalArgumentException when property name is not
+ * empty and the property is not found in configuration
+ **/
+ public static void dumpConfiguration(Configuration config,
+ String propertyName, Writer out) throws IOException {
+ if(Strings.isNullOrEmpty(propertyName)) {
+ dumpConfiguration(config, out);
+ } else if (Strings.isNullOrEmpty(config.get(propertyName))) {
+ throw new IllegalArgumentException("Property " +
+ propertyName + " not found");
+ } else {
+ JsonFactory dumpFactory = new JsonFactory();
+ JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeFieldName("property");
+ appendJSONProperty(dumpGenerator, config, propertyName,
+ new ConfigRedactor(config));
+ dumpGenerator.writeEndObject();
+ dumpGenerator.flush();
+ }
+ }
+
+ /**
+ * Writes out all properties and their attributes (final and resource) to
+ * the given {@link Writer}, the format of the output would be,
+ *
+ * <pre>
+ * { "properties" :
+ * [ { key : "key1",
+ * value : "value1",
+ * isFinal : "key1.isFinal",
+ * resource : "key1.resource" },
+ * { key : "key2",
+ * value : "value2",
+ * isFinal : "ke2.isFinal",
+ * resource : "key2.resource" }
+ * ]
+ * }
+ * </pre>
+ *
+ * It does not output the properties of the configuration object which
+ * is loaded from an input stream.
+ * <p>
+ *
+ * @param config the configuration
+ * @param out the Writer to write to
+ * @throws IOException
+ */
+ public static void dumpConfiguration(Configuration config,
+ Writer out) throws IOException {
+ JsonFactory dumpFactory = new JsonFactory();
+ JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
+ dumpGenerator.writeStartObject();
+ dumpGenerator.writeFieldName("properties");
+ dumpGenerator.writeStartArray();
+ dumpGenerator.flush();
+ ConfigRedactor redactor = new ConfigRedactor(config);
+ synchronized (config) {
+ for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
+ appendJSONProperty(dumpGenerator, config, item.getKey().toString(),
+ redactor);
+ }
+ }
+ dumpGenerator.writeEndArray();
+ dumpGenerator.writeEndObject();
+ dumpGenerator.flush();
+ }
+
+ /**
+ * Write property and its attributes as json format to given
+ * {@link JsonGenerator}.
+ *
+ * @param jsonGen json writer
+ * @param config configuration
+ * @param name property name
+ * @throws IOException
+ */
+ private static void appendJSONProperty(JsonGenerator jsonGen,
+ Configuration config, String name, ConfigRedactor redactor)
+ throws IOException {
+ // skip writing if given property name is empty or null
+ if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
+ jsonGen.writeStartObject();
+ jsonGen.writeStringField("key", name);
+ jsonGen.writeStringField("value",
+ redactor.redact(name, config.get(name)));
+ jsonGen.writeBooleanField("isFinal",
+ config.finalParameters.contains(name));
+ String[] resources = config.updatingResource != null ?
+ config.updatingResource.get(name) : null;
+ String resource = UNKNOWN_RESOURCE;
+ if (resources != null && resources.length > 0) {
+ resource = resources[0];
+ }
+ jsonGen.writeStringField("resource", resource);
+ jsonGen.writeEndObject();
+ }
+ }
+
+ /**
+ * Get the {@link ClassLoader} for this job.
+ *
+ * @return the correct class loader.
+ */
+ public ClassLoader getClassLoader() {
+ return classLoader;
+ }
+
+ /**
+ * Set the class loader that will be used to load the various objects.
+ *
+ * @param classLoader the new class loader.
+ */
+ public void setClassLoader(ClassLoader classLoader) {
+ this.classLoader = classLoader;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Configuration: ");
+ if(loadDefaults) {
+ toString(defaultResources, sb);
+ if(resources.size()>0) {
+ sb.append(", ");
+ }
+ }
+ toString(resources, sb);
+ return sb.toString();
+ }
+
+ private <T> void toString(List<T> resources, StringBuilder sb) {
+ ListIterator<T> i = resources.listIterator();
+ while (i.hasNext()) {
+ if (i.nextIndex() != 0) {
+ sb.append(", ");
+ }
+ sb.append(i.next());
+ }
+ }
+
+ /**
+ * Set the quietness-mode.
+ *
+ * In the quiet-mode, error and informational messages might not be logged.
+ *
+ * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+ * to turn it off.
+ */
+ public synchronized void setQuietMode(boolean quietmode) {
+ this.quietmode = quietmode;
+ }
+
+ synchronized boolean getQuietMode() {
+ return this.quietmode;
+ }
+
+ /** For debugging. List non-default properties to the terminal and exit. */
+ public static void main(String[] args) throws Exception {
+ new Configuration().writeXml(System.out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ clear();
+ int size = WritableUtils.readVInt(in);
+ for(int i=0; i < size; ++i) {
+ String key = org.apache.hadoop.io.Text.readString(in);
+ String value = org.apache.hadoop.io.Text.readString(in);
+ set(key, value);
+ String sources[] = WritableUtils.readCompressedStringArray(in);
+ if (sources != null) {
+ putIntoUpdatingResource(key, sources);
+ }
+ }
+ }
+
+ //@Override
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Properties props = getProps();
+ WritableUtils.writeVInt(out, props.size());
+ for(Map.Entry<Object, Object> item: props.entrySet()) {
+ org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
+ org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
+ WritableUtils.writeCompressedStringArray(out, updatingResource != null ?
+ updatingResource.get(item.getKey()) : null);
+ }
+ }
+
+ /**
+ * get keys matching the the regex
+ * @param regex
+ * @return Map<String,String> with matching keys
+ */
+ public Map<String,String> getValByRegex(String regex) {
+ Pattern p = Pattern.compile(regex);
+
+ Map<String,String> result = new HashMap<String,String>();
+ Matcher m;
+
+ for(Map.Entry<Object,Object> item: getProps().entrySet()) {
+ if (item.getKey() instanceof String &&
+ item.getValue() instanceof String) {
+ m = p.matcher((String)item.getKey());
+ if(m.find()) { // match
+ result.put((String) item.getKey(),
+ substituteVars(getProps().getProperty((String) item.getKey())));
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ * A unique class which is used as a sentinel value in the caching
+ * for getClassByName. {@link Configuration#getClassByNameOrNull(String)}
+ */
+ private static abstract class NegativeCacheSentinel {}
+
+ public static void dumpDeprecatedKeys() {
+ DeprecationContext deprecations = deprecationContext.get();
+ for (Map.Entry<String, DeprecatedKeyInfo> entry :
+ deprecations.getDeprecatedKeyMap().entrySet()) {
+ StringBuilder newKeys = new StringBuilder();
+ for (String newKey : entry.getValue().newKeys) {
+ newKeys.append(newKey).append("\t");
+ }
+ System.out.println(entry.getKey() + "\t" + newKeys.toString());
+ }
+ }
+
+ /**
+ * Returns whether or not a deprecated name has been warned. If the name is not
+ * deprecated then always return false
+ */
+ public static boolean hasWarnedDeprecation(String name) {
+ DeprecationContext deprecations = deprecationContext.get();
+ if(deprecations.getDeprecatedKeyMap().containsKey(name)) {
+ if(deprecations.getDeprecatedKeyMap().get(name).accessed.get()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void putIntoUpdatingResource(String key, String[] value) {
+ Map<String, String[]> localUR = updatingResource;
+ if (localUR == null) {
+ synchronized (this) {
+ localUR = updatingResource;
+ if (localUR == null) {
+ updatingResource = localUR = new ConcurrentHashMap<>(8);
+ }
+ }
+ }
+ localUR.put(key, value);
+ }
+}
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
new file mode 100644
index 0000000..72cab1d
--- /dev/null
+++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+//
+// This class is copied from the Hadoop Source Code (Apache License 2.0)
+// in order to override default behavior i the presence of shading
+//
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+
+@SuppressWarnings("unused")
+public class NativeCodeLoader {
+
+ public static boolean isNativeCodeLoaded() {
+ return false;
+ }
+
+ public static boolean buildSupportsSnappy() {
+ return false;
+ }
+
+ public static boolean buildSupportsOpenssl() {
+ return false;
+ }
+
+ public static boolean buildSupportsIsal() {
+ return false;
+ }
+
+ public static boolean buildSupportsZstd() {
+ return false;
+ }
+
+ public static String getLibraryName() {
+ return null;
+ }
+
+ private NativeCodeLoader() {}
+}
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/util/VersionInfo.java b/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/util/VersionInfo.java
new file mode 100644
index 0000000..0afd723
--- /dev/null
+++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/java/org/apache/hadoop/util/VersionInfo.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.util.Properties;
+
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+//
+// This class is copied from the Hadoop Source Code (Apache License 2.0)
+// in order to override default behavior i the presence of shading
+//
+// ----------------------------------------------------------------------------
+// ----------------------------------------------------------------------------
+
+@SuppressWarnings("all")
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class VersionInfo {
+
+ private final Properties info;
+
+ protected VersionInfo(String component) {
+ info = new Properties();
+
+ if ("common".equals(component)) {
+ info.setProperty("version", "3.1.0");
+ info.setProperty("revision", "16b70619a24cdcf5d3b0fcf4b58ca77238ccbe6d");
+ info.setProperty("branch", "branch-3.1.0");
+ info.setProperty("user", "wtan");
+ info.setProperty("date", "2018-04-03T04:00Z");
+ info.setProperty("url", "git@github.com:hortonworks/hadoop-common-trunk.git");
+ info.setProperty("srcChecksum", "14182d20c972b3e2105580a1ad6990");
+ info.setProperty("protocVersion", "2.5.0");
+ }
+ }
+
+ protected String _getVersion() {
+ return info.getProperty("version", "Unknown");
+ }
+
+ protected String _getRevision() {
+ return info.getProperty("revision", "Unknown");
+ }
+
+ protected String _getBranch() {
+ return info.getProperty("branch", "Unknown");
+ }
+
+ protected String _getDate() {
+ return info.getProperty("date", "Unknown");
+ }
+
+ protected String _getUser() {
+ return info.getProperty("user", "Unknown");
+ }
+
+ protected String _getUrl() {
+ return info.getProperty("url", "Unknown");
+ }
+
+ protected String _getSrcChecksum() {
+ return info.getProperty("srcChecksum", "Unknown");
+ }
+
+ protected String _getBuildVersion(){
+ return _getVersion() +
+ " from " + _getRevision() +
+ " by " + _getUser() +
+ " source checksum " + _getSrcChecksum();
+ }
+
+ protected String _getProtocVersion() {
+ return info.getProperty("protocVersion", "Unknown");
+ }
+
+ private static final VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
+
+ public static String getVersion() {
+ return COMMON_VERSION_INFO._getVersion();
+ }
+
+ public static String getRevision() {
+ return COMMON_VERSION_INFO._getRevision();
+ }
+
+ public static String getBranch() {
+ return COMMON_VERSION_INFO._getBranch();
+ }
+
+ public static String getDate() {
+ return COMMON_VERSION_INFO._getDate();
+ }
+
+ public static String getUser() {
+ return COMMON_VERSION_INFO._getUser();
+ }
+
+ public static String getUrl() {
+ return COMMON_VERSION_INFO._getUrl();
+ }
+
+ public static String getSrcChecksum() {
+ return COMMON_VERSION_INFO._getSrcChecksum();
+ }
+
+ public static String getBuildVersion(){
+ return COMMON_VERSION_INFO._getBuildVersion();
+ }
+
+ public static String getProtocVersion(){
+ return COMMON_VERSION_INFO._getProtocVersion();
+ }
+}
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/core-default-shaded.xml b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/core-default-shaded.xml
new file mode 100644
index 0000000..5b51ff6
--- /dev/null
+++ b/flink-filesystems/flink-fs-hadoop-shaded/src/main/resources/core-default-shaded.xml
@@ -0,0 +1,2775 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Do not modify this file directly. Instead, copy entries that you -->
+<!-- wish to modify from this file into core-site.xml and change them -->
+<!-- there. If core-site.xml does not already exist, create it. -->
+
+<configuration>
+
+ <!--- global properties -->
+
+ <property>
+ <name>hadoop.common.configuration.version</name>
+ <value>3.0.0</value>
+ <description>version of this configuration file</description>
+ </property>
+
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>/tmp/hadoop-${user.name}</value>
+ <description>A base for other temporary directories.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.filter.initializers</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.http.lib.StaticUserWebFilter</value>
+ <description>A comma separated list of class names. Each class in the list
+ must extend org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.http.FilterInitializer. The corresponding
+ Filter will be initialized. Then, the Filter will be applied to all user
+ facing jsp and servlet web pages. The ordering of the list defines the
+ ordering of the filters.</description>
+ </property>
+
+ <!--- security properties -->
+
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ <description>Is service-level authorization enabled?</description>
+ </property>
+
+ <property>
+ <name>hadoop.security.instrumentation.requires.admin</name>
+ <value>false</value>
+ <description>
+ Indicates if administrator ACLs are required to access
+ instrumentation servlets (JMX, METRICS, CONF, STACKS).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ <description>Possible values are simple (no authentication), and kerberos
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback</value>
+ <description>
+ Class for user to group mapping (get groups for a given user) for ACL.
+ The default implementation,
+ org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback,
+ will determine if the Java Native Interface (JNI) is available. If JNI is
+ available the implementation will use the API within hadoop to resolve a
+ list of groups for a user. If JNI is not available then the shell
+ implementation, ShellBasedUnixGroupsMapping, is used. This implementation
+ shells out to the Linux/Unix environment with the
+ <code>bash -c groups</code> command to resolve a list of groups for a user.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.interface</name>
+ <description>
+ The name of the Network Interface from which the service should determine
+ its host name for Kerberos login. e.g. eth2. In a multi-homed environment,
+ the setting can be used to affect the _HOST substitution in the service
+ Kerberos principal. If this configuration value is not set, the service
+ will use its default hostname as returned by
+ InetAddress.getLocalHost().getCanonicalHostName().
+
+ Most clusters will not require this setting.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.nameserver</name>
+ <description>
+ The host name or IP address of the name server (DNS) which a service Node
+ should use to determine its own host name for Kerberos Login. Requires
+ hadoop.security.dns.interface.
+
+ Most clusters will not require this setting.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.log-slow-lookups.enabled</name>
+ <value>false</value>
+ <description>
+ Time name lookups (via SecurityUtil) and log them if they exceed the
+ configured threshold.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.log-slow-lookups.threshold.ms</name>
+ <value>1000</value>
+ <description>
+ If slow lookup logging is enabled, this threshold is used to decide if a
+ lookup is considered slow enough to be logged.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.secs</name>
+ <value>300</value>
+ <description>
+ This is the config controlling the validity of the entries in the cache
+ containing the user->group mapping. When this duration has expired,
+ then the implementation of the group mapping provider is invoked to get
+ the groups of the user and then cached back.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.negative-cache.secs</name>
+ <value>30</value>
+ <description>
+ Expiration time for entries in the the negative user-to-group mapping
+ caching, in seconds. This is useful when invalid users are retrying
+ frequently. It is suggested to set a small value for this expiration, since
+ a transient error in group lookup could temporarily lock out a legitimate
+ user.
+
+ Set this to zero or negative value to disable negative user-to-group caching.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.warn.after.ms</name>
+ <value>5000</value>
+ <description>
+ If looking up a single user to group takes longer than this amount of
+ milliseconds, we will log a warning message.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.background.reload</name>
+ <value>false</value>
+ <description>
+ Whether to reload expired user->group mappings using a background thread
+ pool. If set to true, a pool of
+ hadoop.security.groups.cache.background.reload.threads is created to
+ update the cache in the background.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.background.reload.threads</name>
+ <value>3</value>
+ <description>
+ Only relevant if hadoop.security.groups.cache.background.reload is true.
+ Controls the number of concurrent background user->group cache entry
+ refreshes. Pending refresh requests beyond this value are queued and
+ processed when a thread is free.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.shell.command.timeout</name>
+ <value>0s</value>
+ <description>
+ Used by the ShellBasedUnixGroupsMapping class, this property controls how
+ long to wait for the underlying shell command that is run to fetch groups.
+ Expressed in seconds (e.g. 10s, 1m, etc.), if the running command takes
+ longer than the value configured, the command is aborted and the groups
+ resolver would return a result of no groups found. A value of 0s (default)
+ would mean an infinite wait (i.e. wait until the command exits on its own).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.connection.timeout.ms</name>
+ <value>60000</value>
+ <description>
+ This property is the connection timeout (in milliseconds) for LDAP
+ operations. If the LDAP provider doesn't establish a connection within the
+ specified period, it will abort the connect attempt. Non-positive value
+ means no LDAP connection timeout is specified in which case it waits for the
+ connection to establish until the underlying network times out.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.read.timeout.ms</name>
+ <value>60000</value>
+ <description>
+ This property is the read timeout (in milliseconds) for LDAP
+ operations. If the LDAP provider doesn't get a LDAP response within the
+ specified period, it will abort the read attempt. Non-positive value
+ means no read timeout is specified in which case it waits for the response
+ infinitely.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.url</name>
+ <value></value>
+ <description>
+ The URL of the LDAP server to use for resolving user groups when using
+ the LdapGroupsMapping user to group mapping.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl</name>
+ <value>false</value>
+ <description>
+ Whether or not to use SSL when connecting to the LDAP server.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl.keystore</name>
+ <value></value>
+ <description>
+ File path to the SSL keystore that contains the SSL certificate required
+ by the LDAP server.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl.keystore.password.file</name>
+ <value></value>
+ <description>
+ The path to a file containing the password of the LDAP SSL keystore. If
+ the password is not configured in credential providers and the property
+ hadoop.security.group.mapping.ldap.ssl.keystore.password is not set,
+ LDAPGroupsMapping reads password from the file.
+
+ IMPORTANT: This file should be readable only by the Unix user running
+ the daemons and should be a local file.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl.keystore.password</name>
+ <value></value>
+ <description>
+ The password of the LDAP SSL keystore. this property name is used as an
+ alias to get the password from credential providers. If the password can
+ not be found and hadoop.security.credential.clear-text-fallback is true
+ LDAPGroupsMapping uses the value of this property for password.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.credential.clear-text-fallback</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether or not to fall back to storing credential
+ password as clear text. The default value is true. This property only works
+ when the password can't not be found from credential providers.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.credential.provider.path</name>
+ <value></value>
+ <description>
+ A comma-separated list of URLs that indicates the type and
+ location of a list of providers that should be consulted.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.credstore.java-keystore-provider.password-file</name>
+ <value></value>
+ <description>
+ The path to a file containing the custom password for all keystores
+ that may be configured in the provider path.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.bind.user</name>
+ <value></value>
+ <description>
+ The distinguished name of the user to bind as when connecting to the LDAP
+ server. This may be left blank if the LDAP server supports anonymous binds.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.bind.password.file</name>
+ <value></value>
+ <description>
+ The path to a file containing the password of the bind user. If
+ the password is not configured in credential providers and the property
+ hadoop.security.group.mapping.ldap.bind.password is not set,
+ LDAPGroupsMapping reads password from the file.
+
+ IMPORTANT: This file should be readable only by the Unix user running
+ the daemons and should be a local file.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.bind.password</name>
+ <value></value>
+ <description>
+ The password of the bind user. this property name is used as an
+ alias to get the password from credential providers. If the password can
+ not be found and hadoop.security.credential.clear-text-fallback is true
+ LDAPGroupsMapping uses the value of this property for password.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.base</name>
+ <value></value>
+ <description>
+ The search base for the LDAP connection. This is a distinguished name,
+ and will typically be the root of the LDAP directory.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.userbase</name>
+ <value></value>
+ <description>
+ The search base for the LDAP connection for user search query. This is a
+ distinguished name, and its the root of the LDAP directory for users.
+ If not set, hadoop.security.group.mapping.ldap.base is used.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.groupbase</name>
+ <value></value>
+ <description>
+ The search base for the LDAP connection for group search . This is a
+ distinguished name, and its the root of the LDAP directory for groups.
+ If not set, hadoop.security.group.mapping.ldap.base is used.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.filter.user</name>
+ <value>(&(objectClass=user)(sAMAccountName={0}))</value>
+ <description>
+ An additional filter to use when searching for LDAP users. The default will
+ usually be appropriate for Active Directory installations. If connecting to
+ an LDAP server with a non-AD schema, this should be replaced with
+ (&(objectClass=inetOrgPerson)(uid={0}). {0} is a special string used to
+ denote where the username fits into the filter.
+
+ If the LDAP server supports posixGroups, Hadoop can enable the feature by
+ setting the value of this property to "posixAccount" and the value of
+ the hadoop.security.group.mapping.ldap.search.filter.group property to
+ "posixGroup".
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.filter.group</name>
+ <value>(objectClass=group)</value>
+ <description>
+ An additional filter to use when searching for LDAP groups. This should be
+ changed when resolving groups against a non-Active Directory installation.
+
+ See the description of hadoop.security.group.mapping.ldap.search.filter.user
+ to enable posixGroups support.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.attr.memberof</name>
+ <value></value>
+ <description>
+ The attribute of the user object that identifies its group objects. By
+ default, Hadoop makes two LDAP queries per user if this value is empty. If
+ set, Hadoop will attempt to resolve group names from this attribute,
+ instead of making the second LDAP query to get group objects. The value
+ should be 'memberOf' for an MS AD installation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.attr.member</name>
+ <value>member</value>
+ <description>
+ The attribute of the group object that identifies the users that are
+ members of the group. The default will usually be appropriate for
+ any LDAP installation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.attr.group.name</name>
+ <value>cn</value>
+ <description>
+ The attribute of the group object that identifies the group name. The
+ default will usually be appropriate for all LDAP systems.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.group.hierarchy.levels</name>
+ <value>0</value>
+ <description>
+ The number of levels to go up the group hierarchy when determining
+ which groups a user is part of. 0 Will represent checking just the
+ group that the user belongs to. Each additional level will raise the
+ time it takes to execute a query by at most
+ hadoop.security.group.mapping.ldap.directory.search.timeout.
+ The default will usually be appropriate for all LDAP systems.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.posix.attr.uid.name</name>
+ <value>uidNumber</value>
+ <description>
+ The attribute of posixAccount to use when groups for membership.
+ Mostly useful for schemas wherein groups have memberUids that use an
+ attribute other than uidNumber.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.posix.attr.gid.name</name>
+ <value>gidNumber</value>
+ <description>
+ The attribute of posixAccount indicating the group id.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.directory.search.timeout</name>
+ <value>10000</value>
+ <description>
+ The attribute applied to the LDAP SearchControl properties to set a
+ maximum time limit when searching and awaiting a result.
+ Set to 0 if infinite wait period is desired.
+ Default is 10 seconds. Units in milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.providers</name>
+ <value></value>
+ <description>
+ Comma separated of names of other providers to provide user to group
+ mapping. Used by CompositeGroupsMapping.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.providers.combined</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether groups from the providers are combined or
+ not. The default value is true. If true, then all the providers will be
+ tried to get groups and all the groups are combined to return as the final
+ results. Otherwise, providers are tried one by one in the configured list
+ order, and if any groups are retrieved from any provider, then the groups
+ will be returned without trying the left ones.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.service.user.name.key</name>
+ <value></value>
+ <description>
+ For those cases where the same RPC protocol is implemented by multiple
+ servers, this configuration is required for specifying the principal
+ name to use for the service when the client wishes to make an RPC call.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.user.agent.prefix</name>
+ <value>unknown</value>
+ <description>
+ WASB passes User-Agent header to the Azure back-end. The default value
+ contains WASB version, Java Runtime version, Azure Client library version,
+ and the value of the configuration option fs.azure.user.agent.prefix.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.uid.cache.secs</name>
+ <value>14400</value>
+ <description>
+ This is the config controlling the validity of the entries in the cache
+ containing the userId to userName and groupId to groupName used by
+ NativeIO getFstat().
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>authentication</value>
+ <description>A comma-separated list of protection values for secured sasl
+ connections. Possible values are authentication, integrity and privacy.
+ authentication means authentication only and no integrity or privacy;
+ integrity implies authentication and integrity are enabled; and privacy
+ implies all of authentication, integrity and privacy are enabled.
+ hadoop.security.saslproperties.resolver.class can be used to override
+ the hadoop.rpc.protection for a connection at the server side.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.saslproperties.resolver.class</name>
+ <value></value>
+ <description>SaslPropertiesResolver used to resolve the QOP used for a
+ connection. If not specified, the full set of values specified in
+ hadoop.rpc.protection is used while determining the QOP used for the
+ connection. If a class is specified, then the QOP values returned by
+ the class will be used while determining the QOP used for the connection.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.sensitive-config-keys</name>
+ <value>
+ secret$
+ password$
+ ssl.keystore.pass$
+ fs.s3.*[Ss]ecret.?[Kk]ey
+ fs.s3a.*.server-side-encryption.key
+ fs.azure.account.key.*
+ credential$
+ oauth.*token$
+ hadoop.security.sensitive-config-keys
+ </value>
+ <description>A comma-separated or multi-line list of regular expressions to
+ match configuration keys that should be redacted where appropriate, for
+ example, when logging modified properties during a reconfiguration,
+ private credentials should not be logged.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.workaround.non.threadsafe.getpwuid</name>
+ <value>true</value>
+ <description>Some operating systems or authentication modules are known to
+ have broken implementations of getpwuid_r and getpwgid_r, such that these
+ calls are not thread-safe. Symptoms of this problem include JVM crashes
+ with a stack trace inside these functions. If your system exhibits this
+ issue, enable this configuration parameter to include a lock around the
+ calls as a workaround.
+
+ An incomplete list of some systems known to have this issue is available
+ at http://wiki.apache.org/hadoop/KnownBrokenPwuidImplementations
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kerberos.kinit.command</name>
+ <value>kinit</value>
+ <description>Used to periodically renew Kerberos credentials when provided
+ to Hadoop. The default setting assumes that kinit is in the PATH of users
+ running the Hadoop client. Change this to the absolute path to kinit if this
+ is not the case.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kerberos.min.seconds.before.relogin</name>
+ <value>60</value>
+ <description>The minimum time between relogin attempts for Kerberos, in
+ seconds.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value></value>
+ <description>Maps kerberos principals to local user names</description>
+ </property>
+
+ <property>
+ <name>hadoop.token.files</name>
+ <value></value>
+ <description>List of token cache files that have delegation tokens for hadoop service</description>
+ </property>
+
+ <!-- i/o properties -->
+ <property>
+ <name>io.file.buffer.size</name>
+ <value>4096</value>
+ <description>The size of buffer for use in sequence files.
+ The size of this buffer should probably be a multiple of hardware
+ page size (4096 on Intel x86), and it determines how much data is
+ buffered during read and write operations.</description>
+ </property>
+
+ <property>
+ <name>io.bytes.per.checksum</name>
+ <value>512</value>
+ <description>The number of bytes per checksum. Must not be larger than
+ io.file.buffer.size.</description>
+ </property>
+
+ <property>
+ <name>io.skip.checksum.errors</name>
+ <value>false</value>
+ <description>If true, when a checksum error is encountered while
+ reading a sequence file, entries are skipped, instead of throwing an
+ exception.</description>
+ </property>
+
+ <property>
+ <name>io.compression.codecs</name>
+ <value></value>
+ <description>A comma-separated list of the compression codec classes that can
+ be used for compression/decompression. In addition to any classes specified
+ with this property (which take precedence), codec classes on the classpath
+ are discovered using a Java ServiceLoader.</description>
+ </property>
+
+ <property>
+ <name>io.compression.codec.bzip2.library</name>
+ <value>system-native</value>
+ <description>The native-code library to be used for compression and
+ decompression by the bzip2 codec. This library could be specified
+ either by by name or the full pathname. In the former case, the
+ library is located by the dynamic linker, usually searching the
+ directories specified in the environment variable LD_LIBRARY_PATH.
+
+ The value of "system-native" indicates that the default system
+ library should be used. To indicate that the algorithm should
+ operate entirely in Java, specify "java-builtin".</description>
+ </property>
+
+ <property>
+ <name>io.serializations</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.io.serializer.WritableSerialization, org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value>
+ <description>A list of serialization classes that can be used for
+ obtaining serializers and deserializers.</description>
+ </property>
+
+ <property>
+ <name>io.seqfile.local.dir</name>
+ <value>${hadoop.tmp.dir}/io/local</value>
+ <description>The local directory where sequence file stores intermediate
+ data files during merge. May be a comma-separated list of
+ directories on different devices in order to spread disk i/o.
+ Directories that do not exist are ignored.
+ </description>
+ </property>
+
+ <property>
+ <name>io.map.index.skip</name>
+ <value>0</value>
+ <description>Number of index entries to skip between each entry.
+ Zero by default. Setting this to values larger than zero can
+ facilitate opening large MapFiles using less memory.</description>
+ </property>
+
+ <property>
+ <name>io.map.index.interval</name>
+ <value>128</value>
+ <description>
+ MapFile consist of two files - data file (tuples) and index file
+ (keys). For every io.map.index.interval records written in the
+ data file, an entry (record-key, data-file-position) is written
+ in the index file. This is to allow for doing binary search later
+ within the index file to look up records by their keys and get their
+ closest positions in the data file.
+ </description>
+ </property>
+
+ <property>
+ <name>io.erasurecode.codec.rs.rawcoders</name>
+ <value>rs_native,rs_java</value>
+ <description>
+ Comma separated raw coder implementations for the rs codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
+ </description>
+ </property>
+
+ <property>
+ <name>io.erasurecode.codec.rs-legacy.rawcoders</name>
+ <value>rs-legacy_java</value>
+ <description>
+ Comma separated raw coder implementations for the rs-legacy codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
+ </description>
+ </property>
+
+ <property>
+ <name>io.erasurecode.codec.xor.rawcoders</name>
+ <value>xor_native,xor_java</value>
+ <description>
+ Comma separated raw coder implementations for the xor codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
+ </description>
+ </property>
+
+ <!-- file system properties -->
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>file:///</value>
+ <description>The name of the default file system. A URI whose
+ scheme and authority determine the FileSystem implementation. The
+ uri's scheme determines the config property (fs.SCHEME.impl) naming
+ the FileSystem implementation class. The uri's authority is used to
+ determine the host, port, etc. for a filesystem.</description>
+ </property>
+
+ <property>
+ <name>fs.default.name</name>
+ <value>file:///</value>
+ <description>Deprecated. Use (fs.defaultFS) property
+ instead</description>
+ </property>
+
+ <property>
+ <name>fs.trash.interval</name>
+ <value>0</value>
+ <description>Number of minutes after which the checkpoint
+ gets deleted. If zero, the trash feature is disabled.
+ This option may be configured both on the server and the
+ client. If trash is disabled server side then the client
+ side configuration is checked. If trash is enabled on the
+ server side then the value configured on the server is
+ used and the client configuration value is ignored.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.trash.checkpoint.interval</name>
+ <value>0</value>
+ <description>Number of minutes between trash checkpoints.
+ Should be smaller or equal to fs.trash.interval. If zero,
+ the value is set to the value of fs.trash.interval.
+ Every time the checkpointer runs it creates a new checkpoint
+ out of current and removes checkpoints created more than
+ fs.trash.interval minutes ago.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.protected.directories</name>
+ <value></value>
+ <description>A comma-separated list of directories which cannot
+ be deleted even by the superuser unless they are empty. This
+ setting can be used to guard important system directories
+ against accidental deletion due to administrator error.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.file.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.local.LocalFs</value>
+ <description>The AbstractFileSystem for file: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.har.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.HarFs</value>
+ <description>The AbstractFileSystem for har: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.hdfs.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.Hdfs</value>
+ <description>The FileSystem for hdfs: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.viewfs.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.viewfs.ViewFs</value>
+ <description>The AbstractFileSystem for view file system for viewfs: uris
+ (ie client side mount table:).</description>
+ </property>
+
+ <property>
+ <name>fs.viewfs.rename.strategy</name>
+ <value>SAME_MOUNTPOINT</value>
+ <description>Allowed rename strategy to rename between multiple mountpoints.
+ Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and
+ SAME_FILESYSTEM_ACROSS_MOUNTPOINT.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.ftp.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.ftp.FtpFs</value>
+ <description>The FileSystem for Ftp: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.webhdfs.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.WebHdfs</value>
+ <description>The FileSystem for webhdfs: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.swebhdfs.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.SWebHdfs</value>
+ <description>The FileSystem for swebhdfs: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.ftp.host</name>
+ <value>0.0.0.0</value>
+ <description>FTP filesystem connects to this server</description>
+ </property>
+
+ <property>
+ <name>fs.ftp.host.port</name>
+ <value>21</value>
+ <description>
+ FTP filesystem connects to fs.ftp.host on this port
+ </description>
+ </property>
+
+ <property>
+ <name>fs.ftp.data.connection.mode</name>
+ <value>ACTIVE_LOCAL_DATA_CONNECTION_MODE</value>
+ <description>Set the FTPClient's data connection mode based on configuration.
+ Valid values are ACTIVE_LOCAL_DATA_CONNECTION_MODE,
+ PASSIVE_LOCAL_DATA_CONNECTION_MODE and PASSIVE_REMOTE_DATA_CONNECTION_MODE.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.ftp.transfer.mode</name>
+ <value>BLOCK_TRANSFER_MODE</value>
+ <description>
+ Set FTP's transfer mode based on configuration. Valid values are
+ STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.df.interval</name>
+ <value>60000</value>
+ <description>Disk usage statistics refresh interval in msec.</description>
+ </property>
+
+ <property>
+ <name>fs.du.interval</name>
+ <value>600000</value>
+ <description>File space usage statistics refresh interval in msec.</description>
+ </property>
+
+ <property>
+ <name>fs.swift.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
+ <description>The implementation class of the OpenStack Swift Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.automatic.close</name>
+ <value>true</value>
+ <description>By default, FileSystem instances are automatically closed at program
+ exit using a JVM shutdown hook. Setting this property to false disables this
+ behavior. This is an advanced option that should only be used by server applications
+ requiring a more carefully orchestrated shutdown sequence.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.access.key</name>
+ <description>AWS access key ID used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <description>AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <description>
+ Comma-separated class names of credential provider classes which implement
+ com.amazonaws.auth.AWSCredentialsProvider.
+
+ These are loaded and queried in sequence for a valid set of credentials.
+ Each listed class must implement one of the following means of
+ construction, which are attempted in order:
+ 1. a public constructor accepting java.net.URI and
+ org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.conf.Configuration,
+ 2. a public static method named getInstance that accepts no
+ arguments and returns an instance of
+ com.amazonaws.auth.AWSCredentialsProvider, or
+ 3. a public default constructor.
+
+ Specifying org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
+ anonymous access to a publicly accessible S3 bucket without any credentials.
+ Please note that allowing anonymous access to an S3 bucket compromises
+ security and therefore is unsuitable for most use cases. It can be useful
+ for accessing public data sets without requiring AWS credentials.
+
+ If unspecified, then the default list of credential provider classes,
+ queried in sequence, is:
+ 1. org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.s3a.BasicAWSCredentialsProvider: supports static
+ configuration of AWS access key ID and secret access key. See also
+ fs.s3a.access.key and fs.s3a.secret.key.
+ 2. com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
+ configuration of AWS access key ID and secret access key in
+ environment variables named AWS_ACCESS_KEY_ID and
+ AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
+ 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+ of instance profile credentials if running in an EC2 VM.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.session.token</name>
+ <description>Session token, when using org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+ as one of the providers.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.security.credential.provider.path</name>
+ <value />
+ <description>
+ Optional comma separated list of credential providers, a list
+ which is prepended to that set in hadoop.security.credential.provider.path
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.maximum</name>
+ <value>15</value>
+ <description>Controls the maximum number of simultaneous connections to S3.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.ssl.enabled</name>
+ <value>true</value>
+ <description>Enables or disables SSL connections to S3.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <description>AWS S3 endpoint to connect to. An up-to-date list is
+ provided in the AWS Documentation: regions and endpoints. Without this
+ property, the standard region (s3.amazonaws.com) is assumed.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>false</value>
+ <description>Enable S3 path style access ie disabling the default virtual hosting behaviour.
+ Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual hosting.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.host</name>
+ <description>Hostname of the (optional) proxy server for S3 connections.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.port</name>
+ <description>Proxy server port. If this property is not set
+ but fs.s3a.proxy.host is, port 80 or 443 is assumed (consistent with
+ the value of fs.s3a.connection.ssl.enabled).</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.username</name>
+ <description>Username for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.password</name>
+ <description>Password for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.domain</name>
+ <description>Domain for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.workstation</name>
+ <description>Workstation for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.attempts.maximum</name>
+ <value>20</value>
+ <description>How many times we should retry commands on transient errors.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.establish.timeout</name>
+ <value>5000</value>
+ <description>Socket connection setup timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.timeout</name>
+ <value>200000</value>
+ <description>Socket connection timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.socket.send.buffer</name>
+ <value>8192</value>
+ <description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.socket.recv.buffer</name>
+ <value>8192</value>
+ <description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.paging.maximum</name>
+ <value>5000</value>
+ <description>How many keys to request from S3 when doing
+ directory listings at a time.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.threads.max</name>
+ <value>10</value>
+ <description>The total number of threads available in the filesystem for data
+ uploads *or any other queued filesystem operation*.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.threads.keepalivetime</name>
+ <value>60</value>
+ <description>Number of seconds a thread can be idle before being
+ terminated.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.max.total.tasks</name>
+ <value>5</value>
+ <description>The number of operations which can be queued for execution</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.size</name>
+ <value>100M</value>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.threshold</name>
+ <value>2147483647</value>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ This also controls the partition size in renamed files, as rename() involves
+ copying the source file(s).
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multiobjectdelete.enable</name>
+ <value>true</value>
+ <description>When enabled, multiple single-object delete requests are replaced by
+ a single 'delete multiple objects'-request, reducing the number of requests.
+ Beware: legacy S3-compatible object stores might not support this request.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.acl.default</name>
+ <description>Set a canned ACL for newly created and copied objects. Value may be Private,
+ PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
+ or BucketOwnerFullControl.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.purge</name>
+ <value>false</value>
+ <description>True if you want to purge existing multipart uploads that may not have been
+ completed/aborted correctly. The corresponding purge age is defined in
+ fs.s3a.multipart.purge.age.
+ If set, when the filesystem is instantiated then all outstanding uploads
+ older than the purge age will be terminated -across the entire bucket.
+ This will impact multipart uploads by other applications and users. so should
+ be used sparingly, with an age value chosen to stop failed uploads, without
+ breaking ongoing operations.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.purge.age</name>
+ <value>86400</value>
+ <description>Minimum age in seconds of multipart uploads to purge.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.server-side-encryption-algorithm</name>
+ <description>Specify a server-side encryption algorithm for s3a: file system.
+ Unset by default. It supports the following values: 'AES256' (for SSE-S3),
+ 'SSE-KMS' and 'SSE-C'.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.server-side-encryption.key</name>
+ <description>Specific encryption key to use if fs.s3a.server-side-encryption-algorithm
+ has been set to 'SSE-KMS' or 'SSE-C'. In the case of SSE-C, the value of this property
+ should be the Base64 encoded key. If you are using SSE-KMS and leave this property empty,
+ you'll be using your default's S3 KMS key, otherwise you should set this property to
+ the specific KMS key id.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.signing-algorithm</name>
+ <description>Override the default signing algorithm so legacy
+ implementations can still be used</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.block.size</name>
+ <value>32M</value>
+ <description>Block size to use when reading files using s3a: file system.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.buffer.dir</name>
+ <value>${hadoop.tmp.dir}/s3a</value>
+ <description>Comma separated list of directories that will be used to buffer file
+ uploads to.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.fast.upload.buffer</name>
+ <value>disk</value>
+ <description>
+ The buffering mechanism to for data being written.
+ Values: disk, array, bytebuffer.
+
+ "disk" will use the directories listed in fs.s3a.buffer.dir as
+ the location(s) to save data prior to being uploaded.
+
+ "array" uses arrays in the JVM heap
+
+ "bytebuffer" uses off-heap memory within the JVM.
+
+ Both "array" and "bytebuffer" will consume memory in a single stream up to the number
+ of blocks set by:
+
+ fs.s3a.multipart.size * fs.s3a.fast.upload.active.blocks.
+
+ If using either of these mechanisms, keep this value low
+
+ The total number of threads performing work across all threads is set by
+ fs.s3a.threads.max, with fs.s3a.max.total.tasks values setting the number of queued
+ work items.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.fast.upload.active.blocks</name>
+ <value>4</value>
+ <description>
+ Maximum Number of blocks a single output stream can have
+ active (uploading, or queued to the central FileSystem
+ instance's pool of queued operations.
+
+ This stops a single stream overloading the shared thread pool.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.readahead.range</name>
+ <value>64K</value>
+ <description>Bytes to read ahead during a seek() before closing and
+ re-opening the S3 HTTP connection. This option will be overridden if
+ any call to setReadahead() is made to an open stream.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.user.agent.prefix</name>
+ <value></value>
+ <description>
+ Sets a custom value that will be prepended to the User-Agent header sent in
+ HTTP requests to the S3 back-end by S3AFileSystem. The User-Agent header
+ always includes the Hadoop version number followed by a string generated by
+ the AWS SDK. An example is "User-Agent: Hadoop 2.8.0, aws-sdk-java/1.10.6".
+ If this optional property is set, then its value is prepended to create a
+ customized User-Agent. For example, if this configuration property was set
+ to "MyApp", then an example of the resulting User-Agent would be
+ "User-Agent: MyApp, Hadoop 2.8.0, aws-sdk-java/1.10.6".
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.metadatastore.authoritative</name>
+ <value>false</value>
+ <description>
+ When true, allow MetadataStore implementations to act as source of
+ truth for getting file status and directory listings. Even if this
+ is set to true, MetadataStore implementations may choose not to
+ return authoritative results. If the configured MetadataStore does
+ not support being authoritative, this setting will have no effect.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.metadatastore.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
+ <description>
+ Fully-qualified name of the class that implements the MetadataStore
+ to be used by s3a. The default class, NullMetadataStore, has no
+ effect: s3a will continue to treat the backing S3 service as the one
+ and only source of truth for file and directory metadata.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.cli.prune.age</name>
+ <value>86400000</value>
+ <description>
+ Default age (in milliseconds) after which to prune metadata from the
+ metadatastore when the prune command is run. Can be overridden on the
+ command-line.
+ </description>
+ </property>
+
+
+ <property>
+ <name>fs.s3a.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+ <description>The implementation class of the S3A Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.region</name>
+ <value></value>
+ <description>
+ AWS DynamoDB region to connect to. An up-to-date list is
+ provided in the AWS Documentation: regions and endpoints. Without this
+ property, the S3Guard will operate table in the associated S3 bucket region.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table</name>
+ <value></value>
+ <description>
+ The DynamoDB table name to operate. Without this property, the respective
+ S3 bucket name will be used.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table.create</name>
+ <value>false</value>
+ <description>
+ If true, the S3A client will create the table if it does not already exist.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
+ <value>500</value>
+ <description>
+ Provisioned throughput requirements for read operations in terms of capacity
+ units for the DynamoDB table. This config value will only be used when
+ creating a new DynamoDB table, though later you can manually provision by
+ increasing or decreasing read capacity as needed for existing tables.
+ See DynamoDB documents for more information.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
+ <value>100</value>
+ <description>
+ Provisioned throughput requirements for write operations in terms of
+ capacity units for the DynamoDB table. Refer to related config
+ fs.s3a.s3guard.ddb.table.capacity.read before usage.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.max.retries</name>
+ <value>9</value>
+ <description>
+ Max retries on batched DynamoDB operations before giving up and
+ throwing an IOException. Each retry is delayed with an exponential
+ backoff timer which starts at 100 milliseconds and approximately
+ doubles each time. The minimum wait before throwing an exception is
+ sum(100, 200, 400, 800, .. 100*2^N-1 ) == 100 * ((2^N)-1)
+ So N = 9 yields at least 51.1 seconds (51,100) milliseconds of blocking
+ before throwing an IOException.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.background.sleep</name>
+ <value>25</value>
+ <description>
+ Length (in milliseconds) of pause between each batch of deletes when
+ pruning metadata. Prevents prune operations (which can typically be low
+ priority background operations) from overly interfering with other I/O
+ operations.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.s3a.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.s3a.S3A</value>
+ <description>The implementation class of the S3A AbstractFileSystem.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.list.version</name>
+ <value>2</value>
+ <description>
+ Select which version of the S3 SDK's List Objects API to use. Currently
+ support 2 (default) and 1 (older API).
+ </description>
+ </property>
+
+ <!-- Azure file system properties -->
+ <property>
+ <name>fs.wasb.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
+ <description>The implementation class of the Native Azure Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.wasbs.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure</value>
+ <description>The implementation class of the Secure Native Azure Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.azure.secure.mode</name>
+ <value>false</value>
+ <description>
+ Config flag to identify the mode in which fs.azure.NativeAzureFileSystem needs
+ to run under. Setting it "true" would make fs.azure.NativeAzureFileSystem use
+ SAS keys to communicate with Azure storage.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.local.sas.key.mode</name>
+ <value>false</value>
+ <description>
+ Works in conjuction with fs.azure.secure.mode. Setting this config to true
+ results in fs.azure.NativeAzureFileSystem using the local SAS key generation
+ where the SAS keys are generating in the same process as fs.azure.NativeAzureFileSystem.
+ If fs.azure.secure.mode flag is set to false, this flag has no effect.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.sas.expiry.period</name>
+ <value>90d</value>
+ <description>
+ The default value to be used for expiration period for SAS keys generated.
+ Can use the following suffix (case insensitive):
+ ms(millis), s(sec), m(min), h(hour), d(day)
+ to specify the time (such as 2s, 2m, 1h, etc.).
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.authorization</name>
+ <value>false</value>
+ <description>
+ Config flag to enable authorization support in WASB. Setting it to "true" enables
+ authorization support to WASB. Currently WASB authorization requires a remote service
+ to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url
+ configuration
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.authorization.caching.enable</name>
+ <value>true</value>
+ <description>
+ Config flag to enable caching of authorization results and saskeys in WASB.
+ This flag is relevant only when fs.azure.authorization is enabled.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
+ <value>true</value>
+ <description>
+ Use container saskey for access to all blobs within the container.
+ Blob-specific saskeys are not used when this setting is enabled.
+ This setting provides better performance compared to blob-specific saskeys.
+ </description>
+ </property>
+ <property>
+ <name>io.seqfile.compress.blocksize</name>
+ <value>1000000</value>
+ <description>The minimum block size for compression in block compressed
+ SequenceFiles.
+ </description>
+ </property>
+
+ <property>
+ <name>io.mapfile.bloom.size</name>
+ <value>1048576</value>
+ <description>The size of BloomFilter-s used in BloomMapFile. Each time this many
+ keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter).
+ Larger values minimize the number of filters, which slightly increases the performance,
+ but may waste too much space if the total number of keys is usually much smaller
+ than this number.
+ </description>
+ </property>
+
+ <property>
+ <name>io.mapfile.bloom.error.rate</name>
+ <value>0.005</value>
+ <description>The rate of false positives in BloomFilter-s used in BloomMapFile.
+ As this value decreases, the size of BloomFilter-s increases exponentially. This
+ value is the probability of encountering false positives (default is 0.5%).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.util.hash.type</name>
+ <value>murmur</value>
+ <description>The default implementation of Hash. Currently this can take one of the
+ two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash.
+ </description>
+ </property>
+
+
+ <!-- ipc properties -->
+
+ <property>
+ <name>ipc.client.idlethreshold</name>
+ <value>4000</value>
+ <description>Defines the threshold number of connections after which
+ connections will be inspected for idleness.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.kill.max</name>
+ <value>10</value>
+ <description>Defines the maximum number of clients to disconnect in one go.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connection.maxidletime</name>
+ <value>10000</value>
+ <description>The maximum time in msec after which a client will bring down the
+ connection to the server.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.max.retries</name>
+ <value>10</value>
+ <description>Indicates the number of retries a client will make to establish
+ a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.retry.interval</name>
+ <value>1000</value>
+ <description>Indicates the number of milliseconds a client will wait for
+ before retrying to establish a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.timeout</name>
+ <value>20000</value>
+ <description>Indicates the number of milliseconds a client will wait for the
+ socket to establish a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.max.retries.on.timeouts</name>
+ <value>45</value>
+ <description>Indicates the number of retries a client will make on socket timeout
+ to establish a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.tcpnodelay</name>
+ <value>true</value>
+ <description>Use TCP_NODELAY flag to bypass Nagle's algorithm transmission delays.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.low-latency</name>
+ <value>false</value>
+ <description>Use low-latency QoS markers for IPC connections.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.ping</name>
+ <value>true</value>
+ <description>Send a ping to the server when timeout on reading the response,
+ if set to true. If no failure is detected, the client retries until at least
+ a byte is read or the time given by ipc.client.rpc-timeout.ms is passed.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.ping.interval</name>
+ <value>60000</value>
+ <description>Timeout on waiting response from server, in milliseconds.
+ The client will send ping when the interval is passed without receiving bytes,
+ if ipc.client.ping is set to true.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.rpc-timeout.ms</name>
+ <value>0</value>
+ <description>Timeout on waiting response from server, in milliseconds.
+ If ipc.client.ping is set to true and this rpc-timeout is greater than
+ the value of ipc.ping.interval, the effective value of the rpc-timeout is
+ rounded up to multiple of ipc.ping.interval.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.server.listen.queue.size</name>
+ <value>128</value>
+ <description>Indicates the length of the listen queue for servers accepting
+ client connections.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.server.log.slow.rpc</name>
+ <value>false</value>
+ <description>This setting is useful to troubleshoot performance issues for
+ various services. If this value is set to true then we log requests that
+ fall into 99th percentile as well as increment RpcSlowCalls counter.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.maximum.data.length</name>
+ <value>67108864</value>
+ <description>This indicates the maximum IPC message length (bytes) that can be
+ accepted by the server. Messages larger than this value are rejected by the
+ immediately to avoid possible OOMs. This setting should rarely need to be
+ changed.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.maximum.response.length</name>
+ <value>134217728</value>
+ <description>This indicates the maximum IPC message length (bytes) that can be
+ accepted by the client. Messages larger than this value are rejected
+ immediately to avoid possible OOMs. This setting should rarely need to be
+ changed. Set to 0 to disable.
+ </description>
+ </property>
+
+ <!-- Proxy Configuration -->
+
+ <property>
+ <name>hadoop.security.impersonation.provider.class</name>
+ <value></value>
+ <description>A class which implements ImpersonationProvider interface, used to
+ authorize whether one user can impersonate a specific user.
+ If not specified, the DefaultImpersonationProvider will be used.
+ If a class is specified, then that class will be used to determine
+ the impersonation capability.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.rpc.socket.factory.class.default</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.net.StandardSocketFactory</value>
+ <description> Default SocketFactory to use. This parameter is expected to be
+ formatted as "package.FactoryClassName".
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
+ <value></value>
+ <description> SocketFactory to use to connect to a DFS. If null or empty, use
+ hadoop.rpc.socket.class.default. This socket factory is also used by
+ DFSClient to create sockets to DataNodes.
+ </description>
+ </property>
+
+
+
+ <property>
+ <name>hadoop.socks.server</name>
+ <value></value>
+ <description> Address (host:port) of the SOCKS server to be used by the
+ SocksSocketFactory.
+ </description>
+ </property>
+
+ <!-- Topology Configuration -->
+ <property>
+ <name>net.topology.node.switch.mapping.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.net.ScriptBasedMapping</value>
+ <description> The default implementation of the DNSToSwitchMapping. It
+ invokes a script specified in net.topology.script.file.name to resolve
+ node names. If the value for net.topology.script.file.name is not set, the
+ default value of DEFAULT_RACK is returned for all node names.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.net.NetworkTopology</value>
+ <description> The default implementation of NetworkTopology which is classic three layer one.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.script.file.name</name>
+ <value></value>
+ <description> The script name that should be invoked to resolve DNS names to
+ NetworkTopology names. Example: the script would take host.foo.bar as an
+ argument, and return /rack1 as the output.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.script.number.args</name>
+ <value>100</value>
+ <description> The max number of args that the script configured with
+ net.topology.script.file.name should be run with. Each arg is an
+ IP address.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.table.file.name</name>
+ <value></value>
+ <description> The file name for a topology file, which is used when the
+ net.topology.node.switch.mapping.impl property is set to
+ org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.net.TableMapping. The file format is a two column text
+ file, with columns separated by whitespace. The first column is a DNS or
+ IP address and the second column specifies the rack where the address maps.
+ If no entry corresponding to a host in the cluster is found, then
+ /default-rack is assumed.
+ </description>
+ </property>
+
+ <!-- Local file system -->
+ <property>
+ <name>file.stream-buffer-size</name>
+ <value>4096</value>
+ <description>The size of buffer to stream files.
+ The size of this buffer should probably be a multiple of hardware
+ page size (4096 on Intel x86), and it determines how much data is
+ buffered during read and write operations.</description>
+ </property>
+
+ <property>
+ <name>file.bytes-per-checksum</name>
+ <value>512</value>
+ <description>The number of bytes per checksum. Must not be larger than
+ file.stream-buffer-size</description>
+ </property>
+
+ <property>
+ <name>file.client-write-packet-size</name>
+ <value>65536</value>
+ <description>Packet size for clients to write</description>
+ </property>
+
+ <property>
+ <name>file.blocksize</name>
+ <value>67108864</value>
+ <description>Block size</description>
+ </property>
+
+ <property>
+ <name>file.replication</name>
+ <value>1</value>
+ <description>Replication factor</description>
+ </property>
+
+ <!-- FTP file system -->
+ <property>
+ <name>ftp.stream-buffer-size</name>
+ <value>4096</value>
+ <description>The size of buffer to stream files.
+ The size of this buffer should probably be a multiple of hardware
+ page size (4096 on Intel x86), and it determines how much data is
+ buffered during read and write operations.</description>
+ </property>
+
+ <property>
+ <name>ftp.bytes-per-checksum</name>
+ <value>512</value>
+ <description>The number of bytes per checksum. Must not be larger than
+ ftp.stream-buffer-size</description>
+ </property>
+
+ <property>
+ <name>ftp.client-write-packet-size</name>
+ <value>65536</value>
+ <description>Packet size for clients to write</description>
+ </property>
+
+ <property>
+ <name>ftp.blocksize</name>
+ <value>67108864</value>
+ <description>Block size</description>
+ </property>
+
+ <property>
+ <name>ftp.replication</name>
+ <value>3</value>
+ <description>Replication factor</description>
+ </property>
+
+ <!-- Tfile -->
+
+ <property>
+ <name>tfile.io.chunk.size</name>
+ <value>1048576</value>
+ <description>
+ Value chunk size in bytes. Default to
+ 1MB. Values of the length less than the chunk size is
+ guaranteed to have known value length in read time (See also
+ TFile.Reader.Scanner.Entry.isValueLengthKnown()).
+ </description>
+ </property>
+
+ <property>
+ <name>tfile.fs.output.buffer.size</name>
+ <value>262144</value>
+ <description>
+ Buffer size used for FSDataOutputStream in bytes.
+ </description>
+ </property>
+
+ <property>
+ <name>tfile.fs.input.buffer.size</name>
+ <value>262144</value>
+ <description>
+ Buffer size used for FSDataInputStream in bytes.
+ </description>
+ </property>
+
+ <!-- HTTP web-consoles Authentication -->
+
+ <property>
+ <name>hadoop.http.authentication.type</name>
+ <value>simple</value>
+ <description>
+ Defines authentication used for Oozie HTTP endpoint.
+ Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.token.validity</name>
+ <value>36000</value>
+ <description>
+ Indicates how long (in seconds) an authentication token is valid before it has
+ to be renewed.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.signature.secret.file</name>
+ <value>${user.home}/hadoop-http-auth-signature-secret</value>
+ <description>
+ The signature secret for signing the authentication tokens.
+ The same secret should be used for JT/NN/DN/TT configurations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.cookie.domain</name>
+ <value></value>
+ <description>
+ The domain to use for the HTTP cookie that stores the authentication token.
+ In order to authentiation to work correctly across all Hadoop nodes web-consoles
+ the domain must be correctly set.
+ IMPORTANT: when using IP addresses, browsers ignore cookies with domain settings.
+ For this setting to work properly all nodes in the cluster must be configured
+ to generate URLs with hostname.domain names on it.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description>
+ Indicates if anonymous requests are allowed when using 'simple' authentication.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@LOCALHOST</value>
+ <description>
+ Indicates the Kerberos principal to be used for HTTP endpoint.
+ The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.kerberos.keytab</name>
+ <value>${user.home}/hadoop.keytab</value>
+ <description>
+ Location of the keytab file with the credentials for the principal.
+ Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop.
+ </description>
+ </property>
+
+ <!-- HTTP CORS support -->
+ <property>
+ <name>hadoop.http.cross-origin.enabled</name>
+ <value>false</value>
+ <description>Enable/disable the cross-origin (CORS) filter.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.allowed-origins</name>
+ <value>*</value>
+ <description>Comma separated list of origins that are allowed for web
+ services needing cross-origin (CORS) support. Wildcards (*) and patterns
+ allowed</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.allowed-methods</name>
+ <value>GET,POST,HEAD</value>
+ <description>Comma separated list of methods that are allowed for web
+ services needing cross-origin (CORS) support.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.allowed-headers</name>
+ <value>X-Requested-With,Content-Type,Accept,Origin</value>
+ <description>Comma separated list of headers that are allowed for web
+ services needing cross-origin (CORS) support.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.max-age</name>
+ <value>1800</value>
+ <description>The number of seconds a pre-flighted request can be cached
+ for web services needing cross-origin (CORS) support.</description>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.methods</name>
+ <value></value>
+ <description>
+ List of fencing methods to use for service fencing. May contain
+ builtin methods (eg shell and sshfence) or user-defined method.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.ssh.connect-timeout</name>
+ <value>30000</value>
+ <description>
+ SSH connection timeout, in milliseconds, to use with the builtin
+ sshfence fencer.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.ssh.private-key-files</name>
+ <value></value>
+ <description>
+ The SSH private key files to use with the builtin sshfence fencer.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.quorum</name>
+ <description>
+ A list of ZooKeeper server addresses, separated by commas, that are
+ to be used by the ZKFailoverController in automatic failover.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.session-timeout.ms</name>
+ <value>5000</value>
+ <description>
+ The session timeout to use when the ZKFC connects to ZooKeeper.
+ Setting this value to a lower value implies that server crashes
+ will be detected more quickly, but risks triggering failover too
+ aggressively in the case of a transient error or network blip.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.parent-znode</name>
+ <value>/hadoop-ha</value>
+ <description>
+ The ZooKeeper znode under which the ZK failover controller stores
+ its information. Note that the nameservice ID is automatically
+ appended to this znode, so it is not normally necessary to
+ configure this, even in a federated environment.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.acl</name>
+ <value>world:anyone:rwcda</value>
+ <description>
+ A comma-separated list of ZooKeeper ACLs to apply to the znodes
+ used by automatic failover. These ACLs are specified in the same
+ format as used by the ZooKeeper CLI.
+
+ If the ACL itself contains secrets, you may instead specify a
+ path to a file, prefixed with the '@' symbol, and the value of
+ this configuration will be loaded from within.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.auth</name>
+ <value></value>
+ <description>
+ A comma-separated list of ZooKeeper authentications to add when
+ connecting to ZooKeeper. These are specified in the same format
+ as used by the "addauth" command in the ZK CLI. It is
+ important that the authentications specified here are sufficient
+ to access znodes with the ACL specified in ha.zookeeper.acl.
+
+ If the auths contain secrets, you may instead specify a
+ path to a file, prefixed with the '@' symbol, and the value of
+ this configuration will be loaded from within.
+ </description>
+ </property>
+
+ <!-- Static Web User Filter properties. -->
+ <property>
+ <name>hadoop.http.staticuser.user</name>
+ <value>dr.who</value>
+ <description>
+ The user name to filter as, on static web filters
+ while rendering content. An example use is the HDFS
+ web UI (user to be used for browsing files).
+ </description>
+ </property>
+
+ <!-- SSLFactory configuration -->
+
+ <property>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ <description>
+ The keystores factory to use for retrieving certificates.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ <description>Whether client certificates are required</description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.hostname.verifier</name>
+ <value>DEFAULT</value>
+ <description>
+ The hostname verifier to provide for HttpsURLConnections.
+ Valid values are: DEFAULT, STRICT, STRICT_IE6, DEFAULT_AND_LOCALHOST and
+ ALLOW_ALL
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ <description>
+ Resource file from which ssl server keystore information will be extracted.
+ This file is looked up in the classpath, typically it should be in Hadoop
+ conf/ directory.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ <description>
+ Resource file from which ssl client keystore information will be extracted
+ This file is looked up in the classpath, typically it should be in Hadoop
+ conf/ directory.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.enabled</name>
+ <value>false</value>
+ <description>
+ Deprecated. Use dfs.http.policy and yarn.http.policy instead.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.enabled.protocols</name>
+ <value>TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2</value>
+ <description>
+ The supported SSL protocols.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.jetty.logs.serve.aliases</name>
+ <value>true</value>
+ <description>
+ Enable/Disable aliases serving from jetty
+ </description>
+ </property>
+
+ <property>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
+ <description>
+ The umask used when creating files and directories.
+ Can be in octal or in symbolic. Examples are:
+ "022" (octal for u=rwx,g=r-x,o=r-x in symbolic),
+ or "u=rwx,g=rwx,o=" (symbolic for 007 in octal).
+ </description>
+ </property>
+
+ <!-- ha properties -->
+
+ <property>
+ <name>ha.health-monitor.connect-retry-interval.ms</name>
+ <value>1000</value>
+ <description>
+ How often to retry connecting to the service.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.health-monitor.check-interval.ms</name>
+ <value>1000</value>
+ <description>
+ How often to check the service.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.health-monitor.sleep-after-disconnect.ms</name>
+ <value>1000</value>
+ <description>
+ How long to sleep after an unexpected RPC error.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.health-monitor.rpc-timeout.ms</name>
+ <value>45000</value>
+ <description>
+ Timeout for the actual monitorHealth() calls.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.new-active.rpc-timeout.ms</name>
+ <value>60000</value>
+ <description>
+ Timeout that the FC waits for the new active to become active
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.graceful-fence.rpc-timeout.ms</name>
+ <value>5000</value>
+ <description>
+ Timeout that the FC waits for the old active to go to standby
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.graceful-fence.connection.retries</name>
+ <value>1</value>
+ <description>
+ FC connection retries for graceful fencing
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.cli-check.rpc-timeout.ms</name>
+ <value>20000</value>
+ <description>
+ Timeout that the CLI (manual) FC waits for monitorHealth, getServiceState
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.fallback-to-simple-auth-allowed</name>
+ <value>false</value>
+ <description>
+ When a client is configured to attempt a secure connection, but attempts to
+ connect to an insecure server, that server may instruct the client to
+ switch to SASL SIMPLE (unsecure) authentication. This setting controls
+ whether or not the client will accept this instruction from the server.
+ When false (the default), the client will not allow the fallback to SIMPLE
+ authentication, and will abort the connection.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.client.resolve.remote.symlinks</name>
+ <value>true</value>
+ <description>
+ Whether to resolve symlinks when accessing a remote Hadoop filesystem.
+ Setting this to false causes an exception to be thrown upon encountering
+ a symlink. This setting does not apply to local filesystems, which
+ automatically resolve local symlinks.
+ </description>
+ </property>
+
+ <property>
+ <name>nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ <description>
+ By default, the export can be mounted by any client. The value string
+ contains machine name and access privilege, separated by whitespace
+ characters. The machine name format can be a single host, a Java regular
+ expression, or an IPv4 address. The access privilege uses rw or ro to
+ specify read/write or read-only access of the machines to exports. If the
+ access privilege is not provided, the default is read-only. Entries are separated by ";".
+ For example: "192.168.0.0/22 rw ; host.*\.example\.com ; host1.test.org ro;".
+ Only the NFS gateway needs to restart after this property is updated.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.user.group.static.mapping.overrides</name>
+ <value>dr.who=;</value>
+ <description>
+ Static mapping of user to groups. This will override the groups if
+ available in the system for the specified user. In other words, groups
+ look-up will not happen for these users, instead groups mapped in this
+ configuration will be used.
+ Mapping should be in this format.
+ user1=group1,group2;user2=;user3=group2;
+ Default, "dr.who=;" will consider "dr.who" as user without groups.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.metrics.quantile.enable</name>
+ <value>false</value>
+ <description>
+ Setting this property to true and rpc.metrics.percentiles.intervals
+ to a comma-separated list of the granularity in seconds, the
+ 50/75/90/95/99th percentile latency for rpc queue/processing time in
+ milliseconds are added to rpc metrics.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.metrics.percentiles.intervals</name>
+ <value></value>
+ <description>
+ A comma-separated list of the granularity in seconds for the metrics which
+ describe the 50/75/90/95/99th percentile latency for rpc queue/processing
+ time. The metrics are outputted if rpc.metrics.quantile.enable is set to
+ true.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE</name>
+ <value></value>
+ <description>
+ The prefix for a given crypto codec, contains a comma-separated
+ list of implementation classes for a given crypto codec (eg EXAMPLECIPHERSUITE).
+ The first implementation will be used if available, others are fallbacks.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.codec.classes.aes.ctr.nopadding</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
+ <description>
+ Comma-separated list of crypto codec implementations for AES/CTR/NoPadding.
+ The first implementation will be used if available, others are fallbacks.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.cipher.suite</name>
+ <value>AES/CTR/NoPadding</value>
+ <description>
+ Cipher suite for crypto codec.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.jce.provider</name>
+ <value></value>
+ <description>
+ The JCE provider name used in CryptoCodec.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.buffer.size</name>
+ <value>8192</value>
+ <description>
+ The buffer size used by CryptoInputStream and CryptoOutputStream.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.java.secure.random.algorithm</name>
+ <value>SHA1PRNG</value>
+ <description>
+ The java secure random algorithm.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.secure.random.impl</name>
+ <value></value>
+ <description>
+ Implementation of secure random.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.random.device.file.path</name>
+ <value>/dev/urandom</value>
+ <description>
+ OS security random device file path.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.key.provider.path</name>
+ <description>
+ The KeyProvider to use when managing zone keys, and interacting with
+ encryption keys when reading and writing to an encryption zone.
+ For hdfs clients, the provider path will be same as namenode's
+ provider path.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.har.impl.disable.cache</name>
+ <value>true</value>
+ <description>Don't cache 'har' filesystem instances.</description>
+ </property>
+
+ <!--- KMSClientProvider configurations -->
+ <property>
+ <name>hadoop.security.kms.client.authentication.retry-count</name>
+ <value>1</value>
+ <description>
+ Number of time to retry connecting to KMS on authentication failure
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.size</name>
+ <value>500</value>
+ <description>
+ Size of the EncryptedKeyVersion cache Queue for each key
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.low-watermark</name>
+ <value>0.3f</value>
+ <description>
+ If size of the EncryptedKeyVersion cache Queue falls below the
+ low watermark, this cache queue will be scheduled for a refill
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.num.refill.threads</name>
+ <value>2</value>
+ <description>
+ Number of threads to use for refilling depleted EncryptedKeyVersion
+ cache Queues
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.expiry</name>
+ <value>43200000</value>
+ <description>
+ Cache expiry time for a Key, after which the cache Queue for this
+ key will be dropped. Default = 12hrs
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.timeout</name>
+ <value>60</value>
+ <description>
+ Sets value for KMS client connection timeout, and the read timeout
+ to KMS servers.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.kms.client.failover.sleep.base.millis</name>
+ <value>100</value>
+ <description>
+ Expert only. The time to wait, in milliseconds, between failover
+ attempts increases exponentially as a function of the number of
+ attempts made so far, with a random factor of +/- 50%. This option
+ specifies the base value used in the failover calculation. The
+ first failover will retry immediately. The 2nd failover attempt
+ will delay at least hadoop.security.client.failover.sleep.base.millis
+ milliseconds. And so on.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.kms.client.failover.sleep.max.millis</name>
+ <value>2000</value>
+ <description>
+ Expert only. The time to wait, in milliseconds, between failover
+ attempts increases exponentially as a function of the number of
+ attempts made so far, with a random factor of +/- 50%. This option
+ specifies the maximum value to wait between failovers.
+ Specifically, the time between two failover attempts will not
+ exceed +/- 50% of hadoop.security.client.failover.sleep.max.millis
+ milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.server.max.connections</name>
+ <value>0</value>
+ <description>The maximum number of concurrent connections a server is allowed
+ to accept. If this limit is exceeded, incoming connections will first fill
+ the listen queue and then may go to an OS-specific listen overflow queue.
+ The client may fail or timeout, but the server can avoid running out of file
+ descriptors using this feature. 0 means no limit.
+ </description>
+ </property>
+
+
+ <!-- YARN registry -->
+
+ <property>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
+ <description>
+ Is the registry enabled in the YARN Resource Manager?
+
+ If true, the YARN RM will, as needed.
+ create the user and system paths, and purge
+ service records when containers, application attempts
+ and applications complete.
+
+ If false, the paths must be created by other means,
+ and no automatic cleanup of service records will take place.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.root</name>
+ <value>/registry</value>
+ <description>
+ The root zookeeper node for the registry
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.session.timeout.ms</name>
+ <value>60000</value>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.connection.timeout.ms</name>
+ <value>15000</value>
+ <description>
+ Zookeeper connection timeout in milliseconds
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.retry.times</name>
+ <value>5</value>
+ <description>
+ Zookeeper connection retry count before failing
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.retry.interval.ms</name>
+ <value>1000</value>
+ <description>
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.retry.ceiling.ms</name>
+ <value>60000</value>
+ <description>
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff.
+
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ <description>
+ List of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.secure</name>
+ <value>false</value>
+ <description>
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.system.acls</name>
+ <value>sasl:yarn@, sasl:mapred@, sasl:hdfs@</value>
+ <description>
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+
+ These are given full access to all entries.
+
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.kerberos.realm</name>
+ <value></value>
+ <description>
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+
+ If empty, the default realm of the running process
+ is used.
+
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.jaas.context</name>
+ <value>Client</value>
+ <description>
+ Key to define the JAAS context. Used in secure
+ mode
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.shell.missing.defaultFs.warning</name>
+ <value>false</value>
+ <description>
+ Enable hdfs shell commands to display warnings if (fs.defaultFS) property
+ is not set.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.shell.safely.delete.limit.num.files</name>
+ <value>100</value>
+ <description>Used by -safely option of hadoop fs shell -rm command to avoid
+ accidental deletion of large directories. When enabled, the -rm command
+ requires confirmation if the number of files to be deleted is greater than
+ this limit. The default limit is 100 files. The warning is disabled if
+ the limit is 0 or the -safely is not specified in -rm command.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.client.htrace.sampler.classes</name>
+ <value></value>
+ <description>The class names of the HTrace Samplers to use for Hadoop
+ filesystem clients.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.htrace.span.receiver.classes</name>
+ <value></value>
+ <description>The class names of the Span Receivers to use for Hadoop.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.logs.enabled</name>
+ <value>true</value>
+ <description>
+ Enable the "/logs" endpoint on all Hadoop daemons, which serves local
+ logs, but may be considered a security risk due to it listing the contents
+ of a directory.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.client.resolve.topology.enabled</name>
+ <value>false</value>
+ <description>Whether the client machine will use the class specified by
+ property net.topology.node.switch.mapping.impl to compute the network
+ distance between itself and remote machines of the FileSystem. Additional
+ properties might need to be configured depending on the class specified
+ in net.topology.node.switch.mapping.impl. For example, if
+ org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.net.ScriptBasedMapping is used, a valid script file
+ needs to be specified in net.topology.script.file.name.
+ </description>
+ </property>
+
+
+ <!-- Azure Data Lake File System Configurations -->
+
+ <property>
+ <name>fs.adl.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.adl.AdlFileSystem</value>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.adl.impl</name>
+ <value>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop.fs.adl.Adl</value>
+ </property>
+
+ <property>
+ <name>adl.feature.ownerandgroup.enableupn</name>
+ <value>false</value>
+ <description>
+ When true : User and Group in FileStatus/AclStatus response is
+ represented as user friendly name as per Azure AD profile.
+
+ When false (default) : User and Group in FileStatus/AclStatus
+ response is represented by the unique identifier from Azure AD
+ profile (Object ID as GUID).
+
+ For optimal performance, false is recommended.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
+ <value>ClientCredential</value>
+ <description>
+ Defines Azure Active Directory OAuth2 access token provider type.
+ Supported types are ClientCredential, RefreshToken, MSI, DeviceCode,
+ and Custom.
+ The ClientCredential type requires property fs.adl.oauth2.client.id,
+ fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
+ The RefreshToken type requires property fs.adl.oauth2.client.id and
+ fs.adl.oauth2.refresh.token.
+ The MSI type reads optional property fs.adl.oauth2.msi.port, if specified.
+ The DeviceCode type requires property
+ fs.adl.oauth2.devicecode.clientapp.id.
+ The Custom type requires property fs.adl.oauth2.access.token.provider.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.client.id</name>
+ <value></value>
+ <description>The OAuth2 client id.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.credential</name>
+ <value></value>
+ <description>The OAuth2 access key.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.refresh.url</name>
+ <value></value>
+ <description>The OAuth2 token endpoint.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.refresh.token</name>
+ <value></value>
+ <description>The OAuth2 refresh token.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.access.token.provider</name>
+ <value></value>
+ <description>
+ The class name of the OAuth2 access token provider.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.msi.port</name>
+ <value></value>
+ <description>
+ The localhost port for the MSI token service. This is the port specified
+ when creating the Azure VM. The default, if this setting is not specified,
+ is 50342.
+ Used by MSI token provider.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.devicecode.clientapp.id</name>
+ <value></value>
+ <description>
+ The app id of the AAD native app in whose context the auth request
+ should be made.
+ Used by DeviceCode token provider.
+ </description>
+ </property>
+
+ <!-- Azure Data Lake File System Configurations Ends Here-->
+
+ <property>
+ <name>hadoop.caller.context.enabled</name>
+ <value>false</value>
+ <description>When the feature is enabled, additional fields are written into
+ name-node audit log records for auditing coarse granularity operations.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.max.size</name>
+ <value>128</value>
+ <description>The maximum bytes a caller context string can have. If the
+ passed caller context is longer than this maximum bytes, client will
+ truncate it before sending to server. Note that the server may have a
+ different maximum size, and will truncate the caller context to the
+ maximum size it allows.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.signature.max.size</name>
+ <value>40</value>
+ <description>
+ The caller's signature (optional) is for offline validation. If the
+ signature exceeds the maximum allowed bytes in server, the caller context
+ will be abandoned, in which case the caller context will not be recorded
+ in audit logs.
+ </description>
+ </property>
+ <!-- SequenceFile's Sorter properties -->
+ <property>
+ <name>seq.io.sort.mb</name>
+ <value>100</value>
+ <description>
+ The total amount of buffer memory to use while sorting files,
+ while using SequenceFile.Sorter, in megabytes. By default,
+ gives each merge stream 1MB, which should minimize seeks.
+ </description>
+ </property>
+ <property>
+ <name>seq.io.sort.factor</name>
+ <value>100</value>
+ <description>
+ The number of streams to merge at once while sorting
+ files using SequenceFile.Sorter.
+ This determines the number of open file handles.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.address</name>
+ <!--value>127.0.0.1:2181</value-->
+ <description>Host:Port of the ZooKeeper server to be used.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.num-retries</name>
+ <value>1000</value>
+ <description>Number of tries to connect to ZooKeeper.</description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.retry-interval-ms</name>
+ <value>1000</value>
+ <description>Retry interval in milliseconds when connecting to ZooKeeper.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.timeout-ms</name>
+ <value>10000</value>
+ <description>ZooKeeper session timeout in milliseconds. Session expiration
+ is managed by the ZooKeeper cluster itself, not by the client. This value is
+ used by the cluster to determine when the client's session expires.
+ Expirations happens when the cluster does not hear from the client within
+ the specified session timeout period (i.e. no heartbeat).</description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.acl</name>
+ <value>world:anyone:rwcda</value>
+ <description>ACL's to be used for ZooKeeper znodes.</description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.auth</name>
+ <description>
+ Specify the auths to be used for the ACL's specified in hadoop.zk.acl.
+ This takes a comma-separated list of authentication mechanisms, each of the
+ form 'scheme:auth' (the same syntax used for the 'addAuth' command in
+ the ZK CLI).
+ </description>
+ </property>
+ <property>
+ <name>hadoop.treat.subject.external</name>
+ <value>false</value>
+ <description>
+ When creating UGI with UserGroupInformation(Subject), treat the passed
+ subject external if set to true, and assume the owner of the subject
+ should do the credential renewal.
+
+ When true this property will introduce an incompatible change which
+ may require changes in client code. For more details, see the jiras:
+ HADOOP-13805,HADOOP-13558.
+ </description>
+ </property>
+</configuration>
diff --git a/flink-filesystems/flink-fs-hadoop-shaded/src/test/resources/core-site.xml b/flink-filesystems/flink-fs-hadoop-shaded/src/test/resources/core-site.xml
new file mode 100644
index 0000000..0283de7
--- /dev/null
+++ b/flink-filesystems/flink-fs-hadoop-shaded/src/test/resources/core-site.xml
@@ -0,0 +1,2775 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Do not modify this file directly. Instead, copy entries that you -->
+<!-- wish to modify from this file into core-site.xml and change them -->
+<!-- there. If core-site.xml does not already exist, create it. -->
+
+<configuration>
+
+ <!--- global properties -->
+
+ <property>
+ <name>hadoop.common.configuration.version</name>
+ <value>3.0.0</value>
+ <description>version of this configuration file</description>
+ </property>
+
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>/tmp/hadoop-${user.name}</value>
+ <description>A base for other temporary directories.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.filter.initializers</name>
+ <value>org.apache.hadoop.http.lib.StaticUserWebFilter</value>
+ <description>A comma separated list of class names. Each class in the list
+ must extend org.apache.hadoop.http.FilterInitializer. The corresponding
+ Filter will be initialized. Then, the Filter will be applied to all user
+ facing jsp and servlet web pages. The ordering of the list defines the
+ ordering of the filters.</description>
+ </property>
+
+ <!--- security properties -->
+
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ <description>Is service-level authorization enabled?</description>
+ </property>
+
+ <property>
+ <name>hadoop.security.instrumentation.requires.admin</name>
+ <value>false</value>
+ <description>
+ Indicates if administrator ACLs are required to access
+ instrumentation servlets (JMX, METRICS, CONF, STACKS).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ <description>Possible values are simple (no authentication), and kerberos
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback</value>
+ <description>
+ Class for user to group mapping (get groups for a given user) for ACL.
+ The default implementation,
+ org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback,
+ will determine if the Java Native Interface (JNI) is available. If JNI is
+ available the implementation will use the API within hadoop to resolve a
+ list of groups for a user. If JNI is not available then the shell
+ implementation, ShellBasedUnixGroupsMapping, is used. This implementation
+ shells out to the Linux/Unix environment with the
+ <code>bash -c groups</code> command to resolve a list of groups for a user.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.interface</name>
+ <description>
+ The name of the Network Interface from which the service should determine
+ its host name for Kerberos login. e.g. eth2. In a multi-homed environment,
+ the setting can be used to affect the _HOST substitution in the service
+ Kerberos principal. If this configuration value is not set, the service
+ will use its default hostname as returned by
+ InetAddress.getLocalHost().getCanonicalHostName().
+
+ Most clusters will not require this setting.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.nameserver</name>
+ <description>
+ The host name or IP address of the name server (DNS) which a service Node
+ should use to determine its own host name for Kerberos Login. Requires
+ hadoop.security.dns.interface.
+
+ Most clusters will not require this setting.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.log-slow-lookups.enabled</name>
+ <value>false</value>
+ <description>
+ Time name lookups (via SecurityUtil) and log them if they exceed the
+ configured threshold.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.dns.log-slow-lookups.threshold.ms</name>
+ <value>1000</value>
+ <description>
+ If slow lookup logging is enabled, this threshold is used to decide if a
+ lookup is considered slow enough to be logged.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.secs</name>
+ <value>300</value>
+ <description>
+ This is the config controlling the validity of the entries in the cache
+ containing the user->group mapping. When this duration has expired,
+ then the implementation of the group mapping provider is invoked to get
+ the groups of the user and then cached back.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.negative-cache.secs</name>
+ <value>30</value>
+ <description>
+ Expiration time for entries in the the negative user-to-group mapping
+ caching, in seconds. This is useful when invalid users are retrying
+ frequently. It is suggested to set a small value for this expiration, since
+ a transient error in group lookup could temporarily lock out a legitimate
+ user.
+
+ Set this to zero or negative value to disable negative user-to-group caching.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.warn.after.ms</name>
+ <value>5000</value>
+ <description>
+ If looking up a single user to group takes longer than this amount of
+ milliseconds, we will log a warning message.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.background.reload</name>
+ <value>false</value>
+ <description>
+ Whether to reload expired user->group mappings using a background thread
+ pool. If set to true, a pool of
+ hadoop.security.groups.cache.background.reload.threads is created to
+ update the cache in the background.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.cache.background.reload.threads</name>
+ <value>3</value>
+ <description>
+ Only relevant if hadoop.security.groups.cache.background.reload is true.
+ Controls the number of concurrent background user->group cache entry
+ refreshes. Pending refresh requests beyond this value are queued and
+ processed when a thread is free.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.groups.shell.command.timeout</name>
+ <value>0s</value>
+ <description>
+ Used by the ShellBasedUnixGroupsMapping class, this property controls how
+ long to wait for the underlying shell command that is run to fetch groups.
+ Expressed in seconds (e.g. 10s, 1m, etc.), if the running command takes
+ longer than the value configured, the command is aborted and the groups
+ resolver would return a result of no groups found. A value of 0s (default)
+ would mean an infinite wait (i.e. wait until the command exits on its own).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.connection.timeout.ms</name>
+ <value>60000</value>
+ <description>
+ This property is the connection timeout (in milliseconds) for LDAP
+ operations. If the LDAP provider doesn't establish a connection within the
+ specified period, it will abort the connect attempt. Non-positive value
+ means no LDAP connection timeout is specified in which case it waits for the
+ connection to establish until the underlying network times out.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.read.timeout.ms</name>
+ <value>60000</value>
+ <description>
+ This property is the read timeout (in milliseconds) for LDAP
+ operations. If the LDAP provider doesn't get a LDAP response within the
+ specified period, it will abort the read attempt. Non-positive value
+ means no read timeout is specified in which case it waits for the response
+ infinitely.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.url</name>
+ <value></value>
+ <description>
+ The URL of the LDAP server to use for resolving user groups when using
+ the LdapGroupsMapping user to group mapping.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl</name>
+ <value>false</value>
+ <description>
+ Whether or not to use SSL when connecting to the LDAP server.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl.keystore</name>
+ <value></value>
+ <description>
+ File path to the SSL keystore that contains the SSL certificate required
+ by the LDAP server.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl.keystore.password.file</name>
+ <value></value>
+ <description>
+ The path to a file containing the password of the LDAP SSL keystore. If
+ the password is not configured in credential providers and the property
+ hadoop.security.group.mapping.ldap.ssl.keystore.password is not set,
+ LDAPGroupsMapping reads password from the file.
+
+ IMPORTANT: This file should be readable only by the Unix user running
+ the daemons and should be a local file.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.ssl.keystore.password</name>
+ <value></value>
+ <description>
+ The password of the LDAP SSL keystore. this property name is used as an
+ alias to get the password from credential providers. If the password can
+ not be found and hadoop.security.credential.clear-text-fallback is true
+ LDAPGroupsMapping uses the value of this property for password.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.credential.clear-text-fallback</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether or not to fall back to storing credential
+ password as clear text. The default value is true. This property only works
+ when the password can't not be found from credential providers.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.credential.provider.path</name>
+ <value></value>
+ <description>
+ A comma-separated list of URLs that indicates the type and
+ location of a list of providers that should be consulted.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.credstore.java-keystore-provider.password-file</name>
+ <value></value>
+ <description>
+ The path to a file containing the custom password for all keystores
+ that may be configured in the provider path.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.bind.user</name>
+ <value></value>
+ <description>
+ The distinguished name of the user to bind as when connecting to the LDAP
+ server. This may be left blank if the LDAP server supports anonymous binds.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.bind.password.file</name>
+ <value></value>
+ <description>
+ The path to a file containing the password of the bind user. If
+ the password is not configured in credential providers and the property
+ hadoop.security.group.mapping.ldap.bind.password is not set,
+ LDAPGroupsMapping reads password from the file.
+
+ IMPORTANT: This file should be readable only by the Unix user running
+ the daemons and should be a local file.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.bind.password</name>
+ <value></value>
+ <description>
+ The password of the bind user. this property name is used as an
+ alias to get the password from credential providers. If the password can
+ not be found and hadoop.security.credential.clear-text-fallback is true
+ LDAPGroupsMapping uses the value of this property for password.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.base</name>
+ <value></value>
+ <description>
+ The search base for the LDAP connection. This is a distinguished name,
+ and will typically be the root of the LDAP directory.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.userbase</name>
+ <value></value>
+ <description>
+ The search base for the LDAP connection for user search query. This is a
+ distinguished name, and its the root of the LDAP directory for users.
+ If not set, hadoop.security.group.mapping.ldap.base is used.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.groupbase</name>
+ <value></value>
+ <description>
+ The search base for the LDAP connection for group search . This is a
+ distinguished name, and its the root of the LDAP directory for groups.
+ If not set, hadoop.security.group.mapping.ldap.base is used.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.filter.user</name>
+ <value>(&(objectClass=user)(sAMAccountName={0}))</value>
+ <description>
+ An additional filter to use when searching for LDAP users. The default will
+ usually be appropriate for Active Directory installations. If connecting to
+ an LDAP server with a non-AD schema, this should be replaced with
+ (&(objectClass=inetOrgPerson)(uid={0}). {0} is a special string used to
+ denote where the username fits into the filter.
+
+ If the LDAP server supports posixGroups, Hadoop can enable the feature by
+ setting the value of this property to "posixAccount" and the value of
+ the hadoop.security.group.mapping.ldap.search.filter.group property to
+ "posixGroup".
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.filter.group</name>
+ <value>(objectClass=group)</value>
+ <description>
+ An additional filter to use when searching for LDAP groups. This should be
+ changed when resolving groups against a non-Active Directory installation.
+
+ See the description of hadoop.security.group.mapping.ldap.search.filter.user
+ to enable posixGroups support.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.attr.memberof</name>
+ <value></value>
+ <description>
+ The attribute of the user object that identifies its group objects. By
+ default, Hadoop makes two LDAP queries per user if this value is empty. If
+ set, Hadoop will attempt to resolve group names from this attribute,
+ instead of making the second LDAP query to get group objects. The value
+ should be 'memberOf' for an MS AD installation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.attr.member</name>
+ <value>member</value>
+ <description>
+ The attribute of the group object that identifies the users that are
+ members of the group. The default will usually be appropriate for
+ any LDAP installation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.attr.group.name</name>
+ <value>cn</value>
+ <description>
+ The attribute of the group object that identifies the group name. The
+ default will usually be appropriate for all LDAP systems.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.search.group.hierarchy.levels</name>
+ <value>0</value>
+ <description>
+ The number of levels to go up the group hierarchy when determining
+ which groups a user is part of. 0 Will represent checking just the
+ group that the user belongs to. Each additional level will raise the
+ time it takes to execute a query by at most
+ hadoop.security.group.mapping.ldap.directory.search.timeout.
+ The default will usually be appropriate for all LDAP systems.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.posix.attr.uid.name</name>
+ <value>uidNumber</value>
+ <description>
+ The attribute of posixAccount to use when groups for membership.
+ Mostly useful for schemas wherein groups have memberUids that use an
+ attribute other than uidNumber.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.posix.attr.gid.name</name>
+ <value>gidNumber</value>
+ <description>
+ The attribute of posixAccount indicating the group id.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.ldap.directory.search.timeout</name>
+ <value>10000</value>
+ <description>
+ The attribute applied to the LDAP SearchControl properties to set a
+ maximum time limit when searching and awaiting a result.
+ Set to 0 if infinite wait period is desired.
+ Default is 10 seconds. Units in milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.providers</name>
+ <value></value>
+ <description>
+ Comma separated of names of other providers to provide user to group
+ mapping. Used by CompositeGroupsMapping.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.group.mapping.providers.combined</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether groups from the providers are combined or
+ not. The default value is true. If true, then all the providers will be
+ tried to get groups and all the groups are combined to return as the final
+ results. Otherwise, providers are tried one by one in the configured list
+ order, and if any groups are retrieved from any provider, then the groups
+ will be returned without trying the left ones.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.service.user.name.key</name>
+ <value></value>
+ <description>
+ For those cases where the same RPC protocol is implemented by multiple
+ servers, this configuration is required for specifying the principal
+ name to use for the service when the client wishes to make an RPC call.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.user.agent.prefix</name>
+ <value>unknown</value>
+ <description>
+ WASB passes User-Agent header to the Azure back-end. The default value
+ contains WASB version, Java Runtime version, Azure Client library version,
+ and the value of the configuration option fs.azure.user.agent.prefix.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.uid.cache.secs</name>
+ <value>14400</value>
+ <description>
+ This is the config controlling the validity of the entries in the cache
+ containing the userId to userName and groupId to groupName used by
+ NativeIO getFstat().
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>authentication</value>
+ <description>A comma-separated list of protection values for secured sasl
+ connections. Possible values are authentication, integrity and privacy.
+ authentication means authentication only and no integrity or privacy;
+ integrity implies authentication and integrity are enabled; and privacy
+ implies all of authentication, integrity and privacy are enabled.
+ hadoop.security.saslproperties.resolver.class can be used to override
+ the hadoop.rpc.protection for a connection at the server side.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.saslproperties.resolver.class</name>
+ <value></value>
+ <description>SaslPropertiesResolver used to resolve the QOP used for a
+ connection. If not specified, the full set of values specified in
+ hadoop.rpc.protection is used while determining the QOP used for the
+ connection. If a class is specified, then the QOP values returned by
+ the class will be used while determining the QOP used for the connection.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.sensitive-config-keys</name>
+ <value>
+ secret$
+ password$
+ ssl.keystore.pass$
+ fs.s3.*[Ss]ecret.?[Kk]ey
+ fs.s3a.*.server-side-encryption.key
+ fs.azure.account.key.*
+ credential$
+ oauth.*token$
+ hadoop.security.sensitive-config-keys
+ </value>
+ <description>A comma-separated or multi-line list of regular expressions to
+ match configuration keys that should be redacted where appropriate, for
+ example, when logging modified properties during a reconfiguration,
+ private credentials should not be logged.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.workaround.non.threadsafe.getpwuid</name>
+ <value>true</value>
+ <description>Some operating systems or authentication modules are known to
+ have broken implementations of getpwuid_r and getpwgid_r, such that these
+ calls are not thread-safe. Symptoms of this problem include JVM crashes
+ with a stack trace inside these functions. If your system exhibits this
+ issue, enable this configuration parameter to include a lock around the
+ calls as a workaround.
+
+ An incomplete list of some systems known to have this issue is available
+ at http://wiki.apache.org/hadoop/KnownBrokenPwuidImplementations
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kerberos.kinit.command</name>
+ <value>kinit</value>
+ <description>Used to periodically renew Kerberos credentials when provided
+ to Hadoop. The default setting assumes that kinit is in the PATH of users
+ running the Hadoop client. Change this to the absolute path to kinit if this
+ is not the case.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kerberos.min.seconds.before.relogin</name>
+ <value>60</value>
+ <description>The minimum time between relogin attempts for Kerberos, in
+ seconds.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value></value>
+ <description>Maps kerberos principals to local user names</description>
+ </property>
+
+ <property>
+ <name>hadoop.token.files</name>
+ <value></value>
+ <description>List of token cache files that have delegation tokens for hadoop service</description>
+ </property>
+
+ <!-- i/o properties -->
+ <property>
+ <name>io.file.buffer.size</name>
+ <value>4096</value>
+ <description>The size of buffer for use in sequence files.
+ The size of this buffer should probably be a multiple of hardware
+ page size (4096 on Intel x86), and it determines how much data is
+ buffered during read and write operations.</description>
+ </property>
+
+ <property>
+ <name>io.bytes.per.checksum</name>
+ <value>512</value>
+ <description>The number of bytes per checksum. Must not be larger than
+ io.file.buffer.size.</description>
+ </property>
+
+ <property>
+ <name>io.skip.checksum.errors</name>
+ <value>false</value>
+ <description>If true, when a checksum error is encountered while
+ reading a sequence file, entries are skipped, instead of throwing an
+ exception.</description>
+ </property>
+
+ <property>
+ <name>io.compression.codecs</name>
+ <value></value>
+ <description>A comma-separated list of the compression codec classes that can
+ be used for compression/decompression. In addition to any classes specified
+ with this property (which take precedence), codec classes on the classpath
+ are discovered using a Java ServiceLoader.</description>
+ </property>
+
+ <property>
+ <name>io.compression.codec.bzip2.library</name>
+ <value>system-native</value>
+ <description>The native-code library to be used for compression and
+ decompression by the bzip2 codec. This library could be specified
+ either by by name or the full pathname. In the former case, the
+ library is located by the dynamic linker, usually searching the
+ directories specified in the environment variable LD_LIBRARY_PATH.
+
+ The value of "system-native" indicates that the default system
+ library should be used. To indicate that the algorithm should
+ operate entirely in Java, specify "java-builtin".</description>
+ </property>
+
+ <property>
+ <name>io.serializations</name>
+ <value>org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value>
+ <description>A list of serialization classes that can be used for
+ obtaining serializers and deserializers.</description>
+ </property>
+
+ <property>
+ <name>io.seqfile.local.dir</name>
+ <value>${hadoop.tmp.dir}/io/local</value>
+ <description>The local directory where sequence file stores intermediate
+ data files during merge. May be a comma-separated list of
+ directories on different devices in order to spread disk i/o.
+ Directories that do not exist are ignored.
+ </description>
+ </property>
+
+ <property>
+ <name>io.map.index.skip</name>
+ <value>0</value>
+ <description>Number of index entries to skip between each entry.
+ Zero by default. Setting this to values larger than zero can
+ facilitate opening large MapFiles using less memory.</description>
+ </property>
+
+ <property>
+ <name>io.map.index.interval</name>
+ <value>128</value>
+ <description>
+ MapFile consist of two files - data file (tuples) and index file
+ (keys). For every io.map.index.interval records written in the
+ data file, an entry (record-key, data-file-position) is written
+ in the index file. This is to allow for doing binary search later
+ within the index file to look up records by their keys and get their
+ closest positions in the data file.
+ </description>
+ </property>
+
+ <property>
+ <name>io.erasurecode.codec.rs.rawcoders</name>
+ <value>rs_native,rs_java</value>
+ <description>
+ Comma separated raw coder implementations for the rs codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
+ </description>
+ </property>
+
+ <property>
+ <name>io.erasurecode.codec.rs-legacy.rawcoders</name>
+ <value>rs-legacy_java</value>
+ <description>
+ Comma separated raw coder implementations for the rs-legacy codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
+ </description>
+ </property>
+
+ <property>
+ <name>io.erasurecode.codec.xor.rawcoders</name>
+ <value>xor_native,xor_java</value>
+ <description>
+ Comma separated raw coder implementations for the xor codec. The earlier
+ factory is prior to followings in case of failure of creating raw coders.
+ </description>
+ </property>
+
+ <!-- file system properties -->
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>file:///</value>
+ <description>The name of the default file system. A URI whose
+ scheme and authority determine the FileSystem implementation. The
+ uri's scheme determines the config property (fs.SCHEME.impl) naming
+ the FileSystem implementation class. The uri's authority is used to
+ determine the host, port, etc. for a filesystem.</description>
+ </property>
+
+ <property>
+ <name>fs.default.name</name>
+ <value>file:///</value>
+ <description>Deprecated. Use (fs.defaultFS) property
+ instead</description>
+ </property>
+
+ <property>
+ <name>fs.trash.interval</name>
+ <value>0</value>
+ <description>Number of minutes after which the checkpoint
+ gets deleted. If zero, the trash feature is disabled.
+ This option may be configured both on the server and the
+ client. If trash is disabled server side then the client
+ side configuration is checked. If trash is enabled on the
+ server side then the value configured on the server is
+ used and the client configuration value is ignored.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.trash.checkpoint.interval</name>
+ <value>0</value>
+ <description>Number of minutes between trash checkpoints.
+ Should be smaller or equal to fs.trash.interval. If zero,
+ the value is set to the value of fs.trash.interval.
+ Every time the checkpointer runs it creates a new checkpoint
+ out of current and removes checkpoints created more than
+ fs.trash.interval minutes ago.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.protected.directories</name>
+ <value></value>
+ <description>A comma-separated list of directories which cannot
+ be deleted even by the superuser unless they are empty. This
+ setting can be used to guard important system directories
+ against accidental deletion due to administrator error.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.file.impl</name>
+ <value>org.apache.hadoop.fs.local.LocalFs</value>
+ <description>The AbstractFileSystem for file: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.har.impl</name>
+ <value>org.apache.hadoop.fs.HarFs</value>
+ <description>The AbstractFileSystem for har: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.hdfs.impl</name>
+ <value>org.apache.hadoop.fs.Hdfs</value>
+ <description>The FileSystem for hdfs: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.viewfs.impl</name>
+ <value>org.apache.hadoop.fs.viewfs.ViewFs</value>
+ <description>The AbstractFileSystem for view file system for viewfs: uris
+ (ie client side mount table:).</description>
+ </property>
+
+ <property>
+ <name>fs.viewfs.rename.strategy</name>
+ <value>SAME_MOUNTPOINT</value>
+ <description>Allowed rename strategy to rename between multiple mountpoints.
+ Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and
+ SAME_FILESYSTEM_ACROSS_MOUNTPOINT.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.ftp.impl</name>
+ <value>org.apache.hadoop.fs.ftp.FtpFs</value>
+ <description>The FileSystem for Ftp: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.webhdfs.impl</name>
+ <value>org.apache.hadoop.fs.WebHdfs</value>
+ <description>The FileSystem for webhdfs: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.swebhdfs.impl</name>
+ <value>org.apache.hadoop.fs.SWebHdfs</value>
+ <description>The FileSystem for swebhdfs: uris.</description>
+ </property>
+
+ <property>
+ <name>fs.ftp.host</name>
+ <value>0.0.0.0</value>
+ <description>FTP filesystem connects to this server</description>
+ </property>
+
+ <property>
+ <name>fs.ftp.host.port</name>
+ <value>21</value>
+ <description>
+ FTP filesystem connects to fs.ftp.host on this port
+ </description>
+ </property>
+
+ <property>
+ <name>fs.ftp.data.connection.mode</name>
+ <value>ACTIVE_LOCAL_DATA_CONNECTION_MODE</value>
+ <description>Set the FTPClient's data connection mode based on configuration.
+ Valid values are ACTIVE_LOCAL_DATA_CONNECTION_MODE,
+ PASSIVE_LOCAL_DATA_CONNECTION_MODE and PASSIVE_REMOTE_DATA_CONNECTION_MODE.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.ftp.transfer.mode</name>
+ <value>BLOCK_TRANSFER_MODE</value>
+ <description>
+ Set FTP's transfer mode based on configuration. Valid values are
+ STREAM_TRANSFER_MODE, BLOCK_TRANSFER_MODE and COMPRESSED_TRANSFER_MODE.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.df.interval</name>
+ <value>60000</value>
+ <description>Disk usage statistics refresh interval in msec.</description>
+ </property>
+
+ <property>
+ <name>fs.du.interval</name>
+ <value>600000</value>
+ <description>File space usage statistics refresh interval in msec.</description>
+ </property>
+
+ <property>
+ <name>fs.swift.impl</name>
+ <value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
+ <description>The implementation class of the OpenStack Swift Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.automatic.close</name>
+ <value>true</value>
+ <description>By default, FileSystem instances are automatically closed at program
+ exit using a JVM shutdown hook. Setting this property to false disables this
+ behavior. This is an advanced option that should only be used by server applications
+ requiring a more carefully orchestrated shutdown sequence.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.access.key</name>
+ <description>AWS access key ID used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <description>AWS secret key used by S3A file system. Omit for IAM role-based or provider-based authentication.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <description>
+ Comma-separated class names of credential provider classes which implement
+ com.amazonaws.auth.AWSCredentialsProvider.
+
+ These are loaded and queried in sequence for a valid set of credentials.
+ Each listed class must implement one of the following means of
+ construction, which are attempted in order:
+ 1. a public constructor accepting java.net.URI and
+ org.apache.hadoop.conf.Configuration,
+ 2. a public static method named getInstance that accepts no
+ arguments and returns an instance of
+ com.amazonaws.auth.AWSCredentialsProvider, or
+ 3. a public default constructor.
+
+ Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
+ anonymous access to a publicly accessible S3 bucket without any credentials.
+ Please note that allowing anonymous access to an S3 bucket compromises
+ security and therefore is unsuitable for most use cases. It can be useful
+ for accessing public data sets without requiring AWS credentials.
+
+ If unspecified, then the default list of credential provider classes,
+ queried in sequence, is:
+ 1. org.apache.hadoop.fs.s3a.BasicAWSCredentialsProvider: supports static
+ configuration of AWS access key ID and secret access key. See also
+ fs.s3a.access.key and fs.s3a.secret.key.
+ 2. com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
+ configuration of AWS access key ID and secret access key in
+ environment variables named AWS_ACCESS_KEY_ID and
+ AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
+ 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+ of instance profile credentials if running in an EC2 VM.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.session.token</name>
+ <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+ as one of the providers.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.security.credential.provider.path</name>
+ <value />
+ <description>
+ Optional comma separated list of credential providers, a list
+ which is prepended to that set in hadoop.security.credential.provider.path
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.maximum</name>
+ <value>15</value>
+ <description>Controls the maximum number of simultaneous connections to S3.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.ssl.enabled</name>
+ <value>true</value>
+ <description>Enables or disables SSL connections to S3.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <description>AWS S3 endpoint to connect to. An up-to-date list is
+ provided in the AWS Documentation: regions and endpoints. Without this
+ property, the standard region (s3.amazonaws.com) is assumed.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>false</value>
+ <description>Enable S3 path style access ie disabling the default virtual hosting behaviour.
+ Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual hosting.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.host</name>
+ <description>Hostname of the (optional) proxy server for S3 connections.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.port</name>
+ <description>Proxy server port. If this property is not set
+ but fs.s3a.proxy.host is, port 80 or 443 is assumed (consistent with
+ the value of fs.s3a.connection.ssl.enabled).</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.username</name>
+ <description>Username for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.password</name>
+ <description>Password for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.domain</name>
+ <description>Domain for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.workstation</name>
+ <description>Workstation for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.attempts.maximum</name>
+ <value>20</value>
+ <description>How many times we should retry commands on transient errors.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.establish.timeout</name>
+ <value>5000</value>
+ <description>Socket connection setup timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.connection.timeout</name>
+ <value>200000</value>
+ <description>Socket connection timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.socket.send.buffer</name>
+ <value>8192</value>
+ <description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.socket.recv.buffer</name>
+ <value>8192</value>
+ <description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.paging.maximum</name>
+ <value>5000</value>
+ <description>How many keys to request from S3 when doing
+ directory listings at a time.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.threads.max</name>
+ <value>10</value>
+ <description>The total number of threads available in the filesystem for data
+ uploads *or any other queued filesystem operation*.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.threads.keepalivetime</name>
+ <value>60</value>
+ <description>Number of seconds a thread can be idle before being
+ terminated.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.max.total.tasks</name>
+ <value>5</value>
+ <description>The number of operations which can be queued for execution</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.size</name>
+ <value>100M</value>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.threshold</name>
+ <value>2147483647</value>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ This also controls the partition size in renamed files, as rename() involves
+ copying the source file(s).
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multiobjectdelete.enable</name>
+ <value>true</value>
+ <description>When enabled, multiple single-object delete requests are replaced by
+ a single 'delete multiple objects'-request, reducing the number of requests.
+ Beware: legacy S3-compatible object stores might not support this request.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.acl.default</name>
+ <description>Set a canned ACL for newly created and copied objects. Value may be Private,
+ PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
+ or BucketOwnerFullControl.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.purge</name>
+ <value>false</value>
+ <description>True if you want to purge existing multipart uploads that may not have been
+ completed/aborted correctly. The corresponding purge age is defined in
+ fs.s3a.multipart.purge.age.
+ If set, when the filesystem is instantiated then all outstanding uploads
+ older than the purge age will be terminated -across the entire bucket.
+ This will impact multipart uploads by other applications and users. so should
+ be used sparingly, with an age value chosen to stop failed uploads, without
+ breaking ongoing operations.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.multipart.purge.age</name>
+ <value>86400</value>
+ <description>Minimum age in seconds of multipart uploads to purge.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.server-side-encryption-algorithm</name>
+ <description>Specify a server-side encryption algorithm for s3a: file system.
+ Unset by default. It supports the following values: 'AES256' (for SSE-S3),
+ 'SSE-KMS' and 'SSE-C'.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.server-side-encryption.key</name>
+ <description>Specific encryption key to use if fs.s3a.server-side-encryption-algorithm
+ has been set to 'SSE-KMS' or 'SSE-C'. In the case of SSE-C, the value of this property
+ should be the Base64 encoded key. If you are using SSE-KMS and leave this property empty,
+ you'll be using your default's S3 KMS key, otherwise you should set this property to
+ the specific KMS key id.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.signing-algorithm</name>
+ <description>Override the default signing algorithm so legacy
+ implementations can still be used</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.block.size</name>
+ <value>32M</value>
+ <description>Block size to use when reading files using s3a: file system.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.buffer.dir</name>
+ <value>${hadoop.tmp.dir}/s3a</value>
+ <description>Comma separated list of directories that will be used to buffer file
+ uploads to.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.fast.upload.buffer</name>
+ <value>disk</value>
+ <description>
+ The buffering mechanism to for data being written.
+ Values: disk, array, bytebuffer.
+
+ "disk" will use the directories listed in fs.s3a.buffer.dir as
+ the location(s) to save data prior to being uploaded.
+
+ "array" uses arrays in the JVM heap
+
+ "bytebuffer" uses off-heap memory within the JVM.
+
+ Both "array" and "bytebuffer" will consume memory in a single stream up to the number
+ of blocks set by:
+
+ fs.s3a.multipart.size * fs.s3a.fast.upload.active.blocks.
+
+ If using either of these mechanisms, keep this value low
+
+ The total number of threads performing work across all threads is set by
+ fs.s3a.threads.max, with fs.s3a.max.total.tasks values setting the number of queued
+ work items.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.fast.upload.active.blocks</name>
+ <value>4</value>
+ <description>
+ Maximum Number of blocks a single output stream can have
+ active (uploading, or queued to the central FileSystem
+ instance's pool of queued operations.
+
+ This stops a single stream overloading the shared thread pool.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.readahead.range</name>
+ <value>64K</value>
+ <description>Bytes to read ahead during a seek() before closing and
+ re-opening the S3 HTTP connection. This option will be overridden if
+ any call to setReadahead() is made to an open stream.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.user.agent.prefix</name>
+ <value></value>
+ <description>
+ Sets a custom value that will be prepended to the User-Agent header sent in
+ HTTP requests to the S3 back-end by S3AFileSystem. The User-Agent header
+ always includes the Hadoop version number followed by a string generated by
+ the AWS SDK. An example is "User-Agent: Hadoop 2.8.0, aws-sdk-java/1.10.6".
+ If this optional property is set, then its value is prepended to create a
+ customized User-Agent. For example, if this configuration property was set
+ to "MyApp", then an example of the resulting User-Agent would be
+ "User-Agent: MyApp, Hadoop 2.8.0, aws-sdk-java/1.10.6".
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.metadatastore.authoritative</name>
+ <value>false</value>
+ <description>
+ When true, allow MetadataStore implementations to act as source of
+ truth for getting file status and directory listings. Even if this
+ is set to true, MetadataStore implementations may choose not to
+ return authoritative results. If the configured MetadataStore does
+ not support being authoritative, this setting will have no effect.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.metadatastore.impl</name>
+ <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
+ <description>
+ Fully-qualified name of the class that implements the MetadataStore
+ to be used by s3a. The default class, NullMetadataStore, has no
+ effect: s3a will continue to treat the backing S3 service as the one
+ and only source of truth for file and directory metadata.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.cli.prune.age</name>
+ <value>86400000</value>
+ <description>
+ Default age (in milliseconds) after which to prune metadata from the
+ metadatastore when the prune command is run. Can be overridden on the
+ command-line.
+ </description>
+ </property>
+
+
+ <property>
+ <name>fs.s3a.impl</name>
+ <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
+ <description>The implementation class of the S3A Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.region</name>
+ <value></value>
+ <description>
+ AWS DynamoDB region to connect to. An up-to-date list is
+ provided in the AWS Documentation: regions and endpoints. Without this
+ property, the S3Guard will operate table in the associated S3 bucket region.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table</name>
+ <value></value>
+ <description>
+ The DynamoDB table name to operate. Without this property, the respective
+ S3 bucket name will be used.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table.create</name>
+ <value>false</value>
+ <description>
+ If true, the S3A client will create the table if it does not already exist.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
+ <value>500</value>
+ <description>
+ Provisioned throughput requirements for read operations in terms of capacity
+ units for the DynamoDB table. This config value will only be used when
+ creating a new DynamoDB table, though later you can manually provision by
+ increasing or decreasing read capacity as needed for existing tables.
+ See DynamoDB documents for more information.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
+ <value>100</value>
+ <description>
+ Provisioned throughput requirements for write operations in terms of
+ capacity units for the DynamoDB table. Refer to related config
+ fs.s3a.s3guard.ddb.table.capacity.read before usage.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.max.retries</name>
+ <value>9</value>
+ <description>
+ Max retries on batched DynamoDB operations before giving up and
+ throwing an IOException. Each retry is delayed with an exponential
+ backoff timer which starts at 100 milliseconds and approximately
+ doubles each time. The minimum wait before throwing an exception is
+ sum(100, 200, 400, 800, .. 100*2^N-1 ) == 100 * ((2^N)-1)
+ So N = 9 yields at least 51.1 seconds (51,100) milliseconds of blocking
+ before throwing an IOException.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.s3guard.ddb.background.sleep</name>
+ <value>25</value>
+ <description>
+ Length (in milliseconds) of pause between each batch of deletes when
+ pruning metadata. Prevents prune operations (which can typically be low
+ priority background operations) from overly interfering with other I/O
+ operations.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.s3a.impl</name>
+ <value>org.apache.hadoop.fs.s3a.S3A</value>
+ <description>The implementation class of the S3A AbstractFileSystem.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.list.version</name>
+ <value>2</value>
+ <description>
+ Select which version of the S3 SDK's List Objects API to use. Currently
+ support 2 (default) and 1 (older API).
+ </description>
+ </property>
+
+ <!-- Azure file system properties -->
+ <property>
+ <name>fs.wasb.impl</name>
+ <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
+ <description>The implementation class of the Native Azure Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.wasbs.impl</name>
+ <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure</value>
+ <description>The implementation class of the Secure Native Azure Filesystem</description>
+ </property>
+
+ <property>
+ <name>fs.azure.secure.mode</name>
+ <value>false</value>
+ <description>
+ Config flag to identify the mode in which fs.azure.NativeAzureFileSystem needs
+ to run under. Setting it "true" would make fs.azure.NativeAzureFileSystem use
+ SAS keys to communicate with Azure storage.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.local.sas.key.mode</name>
+ <value>false</value>
+ <description>
+ Works in conjuction with fs.azure.secure.mode. Setting this config to true
+ results in fs.azure.NativeAzureFileSystem using the local SAS key generation
+ where the SAS keys are generating in the same process as fs.azure.NativeAzureFileSystem.
+ If fs.azure.secure.mode flag is set to false, this flag has no effect.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.sas.expiry.period</name>
+ <value>90d</value>
+ <description>
+ The default value to be used for expiration period for SAS keys generated.
+ Can use the following suffix (case insensitive):
+ ms(millis), s(sec), m(min), h(hour), d(day)
+ to specify the time (such as 2s, 2m, 1h, etc.).
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.authorization</name>
+ <value>false</value>
+ <description>
+ Config flag to enable authorization support in WASB. Setting it to "true" enables
+ authorization support to WASB. Currently WASB authorization requires a remote service
+ to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url
+ configuration
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.authorization.caching.enable</name>
+ <value>true</value>
+ <description>
+ Config flag to enable caching of authorization results and saskeys in WASB.
+ This flag is relevant only when fs.azure.authorization is enabled.
+ </description>
+ </property>
+ <property>
+ <name>fs.azure.saskey.usecontainersaskeyforallaccess</name>
+ <value>true</value>
+ <description>
+ Use container saskey for access to all blobs within the container.
+ Blob-specific saskeys are not used when this setting is enabled.
+ This setting provides better performance compared to blob-specific saskeys.
+ </description>
+ </property>
+ <property>
+ <name>io.seqfile.compress.blocksize</name>
+ <value>1000000</value>
+ <description>The minimum block size for compression in block compressed
+ SequenceFiles.
+ </description>
+ </property>
+
+ <property>
+ <name>io.mapfile.bloom.size</name>
+ <value>1048576</value>
+ <description>The size of BloomFilter-s used in BloomMapFile. Each time this many
+ keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter).
+ Larger values minimize the number of filters, which slightly increases the performance,
+ but may waste too much space if the total number of keys is usually much smaller
+ than this number.
+ </description>
+ </property>
+
+ <property>
+ <name>io.mapfile.bloom.error.rate</name>
+ <value>0.005</value>
+ <description>The rate of false positives in BloomFilter-s used in BloomMapFile.
+ As this value decreases, the size of BloomFilter-s increases exponentially. This
+ value is the probability of encountering false positives (default is 0.5%).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.util.hash.type</name>
+ <value>murmur</value>
+ <description>The default implementation of Hash. Currently this can take one of the
+ two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash.
+ </description>
+ </property>
+
+
+ <!-- ipc properties -->
+
+ <property>
+ <name>ipc.client.idlethreshold</name>
+ <value>4000</value>
+ <description>Defines the threshold number of connections after which
+ connections will be inspected for idleness.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.kill.max</name>
+ <value>10</value>
+ <description>Defines the maximum number of clients to disconnect in one go.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connection.maxidletime</name>
+ <value>10000</value>
+ <description>The maximum time in msec after which a client will bring down the
+ connection to the server.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.max.retries</name>
+ <value>10</value>
+ <description>Indicates the number of retries a client will make to establish
+ a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.retry.interval</name>
+ <value>1000</value>
+ <description>Indicates the number of milliseconds a client will wait for
+ before retrying to establish a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.timeout</name>
+ <value>20000</value>
+ <description>Indicates the number of milliseconds a client will wait for the
+ socket to establish a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.max.retries.on.timeouts</name>
+ <value>45</value>
+ <description>Indicates the number of retries a client will make on socket timeout
+ to establish a server connection.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.tcpnodelay</name>
+ <value>true</value>
+ <description>Use TCP_NODELAY flag to bypass Nagle's algorithm transmission delays.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.low-latency</name>
+ <value>false</value>
+ <description>Use low-latency QoS markers for IPC connections.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.ping</name>
+ <value>true</value>
+ <description>Send a ping to the server when timeout on reading the response,
+ if set to true. If no failure is detected, the client retries until at least
+ a byte is read or the time given by ipc.client.rpc-timeout.ms is passed.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.ping.interval</name>
+ <value>60000</value>
+ <description>Timeout on waiting response from server, in milliseconds.
+ The client will send ping when the interval is passed without receiving bytes,
+ if ipc.client.ping is set to true.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.rpc-timeout.ms</name>
+ <value>0</value>
+ <description>Timeout on waiting response from server, in milliseconds.
+ If ipc.client.ping is set to true and this rpc-timeout is greater than
+ the value of ipc.ping.interval, the effective value of the rpc-timeout is
+ rounded up to multiple of ipc.ping.interval.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.server.listen.queue.size</name>
+ <value>128</value>
+ <description>Indicates the length of the listen queue for servers accepting
+ client connections.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.server.log.slow.rpc</name>
+ <value>false</value>
+ <description>This setting is useful to troubleshoot performance issues for
+ various services. If this value is set to true then we log requests that
+ fall into 99th percentile as well as increment RpcSlowCalls counter.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.maximum.data.length</name>
+ <value>67108864</value>
+ <description>This indicates the maximum IPC message length (bytes) that can be
+ accepted by the server. Messages larger than this value are rejected by the
+ immediately to avoid possible OOMs. This setting should rarely need to be
+ changed.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.maximum.response.length</name>
+ <value>134217728</value>
+ <description>This indicates the maximum IPC message length (bytes) that can be
+ accepted by the client. Messages larger than this value are rejected
+ immediately to avoid possible OOMs. This setting should rarely need to be
+ changed. Set to 0 to disable.
+ </description>
+ </property>
+
+ <!-- Proxy Configuration -->
+
+ <property>
+ <name>hadoop.security.impersonation.provider.class</name>
+ <value></value>
+ <description>A class which implements ImpersonationProvider interface, used to
+ authorize whether one user can impersonate a specific user.
+ If not specified, the DefaultImpersonationProvider will be used.
+ If a class is specified, then that class will be used to determine
+ the impersonation capability.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.rpc.socket.factory.class.default</name>
+ <value>org.apache.hadoop.net.StandardSocketFactory</value>
+ <description> Default SocketFactory to use. This parameter is expected to be
+ formatted as "package.FactoryClassName".
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
+ <value></value>
+ <description> SocketFactory to use to connect to a DFS. If null or empty, use
+ hadoop.rpc.socket.class.default. This socket factory is also used by
+ DFSClient to create sockets to DataNodes.
+ </description>
+ </property>
+
+
+
+ <property>
+ <name>hadoop.socks.server</name>
+ <value></value>
+ <description> Address (host:port) of the SOCKS server to be used by the
+ SocksSocketFactory.
+ </description>
+ </property>
+
+ <!-- Topology Configuration -->
+ <property>
+ <name>net.topology.node.switch.mapping.impl</name>
+ <value>org.apache.hadoop.net.ScriptBasedMapping</value>
+ <description> The default implementation of the DNSToSwitchMapping. It
+ invokes a script specified in net.topology.script.file.name to resolve
+ node names. If the value for net.topology.script.file.name is not set, the
+ default value of DEFAULT_RACK is returned for all node names.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.impl</name>
+ <value>org.apache.hadoop.net.NetworkTopology</value>
+ <description> The default implementation of NetworkTopology which is classic three layer one.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.script.file.name</name>
+ <value></value>
+ <description> The script name that should be invoked to resolve DNS names to
+ NetworkTopology names. Example: the script would take host.foo.bar as an
+ argument, and return /rack1 as the output.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.script.number.args</name>
+ <value>100</value>
+ <description> The max number of args that the script configured with
+ net.topology.script.file.name should be run with. Each arg is an
+ IP address.
+ </description>
+ </property>
+
+ <property>
+ <name>net.topology.table.file.name</name>
+ <value></value>
+ <description> The file name for a topology file, which is used when the
+ net.topology.node.switch.mapping.impl property is set to
+ org.apache.hadoop.net.TableMapping. The file format is a two column text
+ file, with columns separated by whitespace. The first column is a DNS or
+ IP address and the second column specifies the rack where the address maps.
+ If no entry corresponding to a host in the cluster is found, then
+ /default-rack is assumed.
+ </description>
+ </property>
+
+ <!-- Local file system -->
+ <property>
+ <name>file.stream-buffer-size</name>
+ <value>4096</value>
+ <description>The size of buffer to stream files.
+ The size of this buffer should probably be a multiple of hardware
+ page size (4096 on Intel x86), and it determines how much data is
+ buffered during read and write operations.</description>
+ </property>
+
+ <property>
+ <name>file.bytes-per-checksum</name>
+ <value>512</value>
+ <description>The number of bytes per checksum. Must not be larger than
+ file.stream-buffer-size</description>
+ </property>
+
+ <property>
+ <name>file.client-write-packet-size</name>
+ <value>65536</value>
+ <description>Packet size for clients to write</description>
+ </property>
+
+ <property>
+ <name>file.blocksize</name>
+ <value>67108864</value>
+ <description>Block size</description>
+ </property>
+
+ <property>
+ <name>file.replication</name>
+ <value>1</value>
+ <description>Replication factor</description>
+ </property>
+
+ <!-- FTP file system -->
+ <property>
+ <name>ftp.stream-buffer-size</name>
+ <value>4096</value>
+ <description>The size of buffer to stream files.
+ The size of this buffer should probably be a multiple of hardware
+ page size (4096 on Intel x86), and it determines how much data is
+ buffered during read and write operations.</description>
+ </property>
+
+ <property>
+ <name>ftp.bytes-per-checksum</name>
+ <value>512</value>
+ <description>The number of bytes per checksum. Must not be larger than
+ ftp.stream-buffer-size</description>
+ </property>
+
+ <property>
+ <name>ftp.client-write-packet-size</name>
+ <value>65536</value>
+ <description>Packet size for clients to write</description>
+ </property>
+
+ <property>
+ <name>ftp.blocksize</name>
+ <value>67108864</value>
+ <description>Block size</description>
+ </property>
+
+ <property>
+ <name>ftp.replication</name>
+ <value>3</value>
+ <description>Replication factor</description>
+ </property>
+
+ <!-- Tfile -->
+
+ <property>
+ <name>tfile.io.chunk.size</name>
+ <value>1048576</value>
+ <description>
+ Value chunk size in bytes. Default to
+ 1MB. Values of the length less than the chunk size is
+ guaranteed to have known value length in read time (See also
+ TFile.Reader.Scanner.Entry.isValueLengthKnown()).
+ </description>
+ </property>
+
+ <property>
+ <name>tfile.fs.output.buffer.size</name>
+ <value>262144</value>
+ <description>
+ Buffer size used for FSDataOutputStream in bytes.
+ </description>
+ </property>
+
+ <property>
+ <name>tfile.fs.input.buffer.size</name>
+ <value>262144</value>
+ <description>
+ Buffer size used for FSDataInputStream in bytes.
+ </description>
+ </property>
+
+ <!-- HTTP web-consoles Authentication -->
+
+ <property>
+ <name>hadoop.http.authentication.type</name>
+ <value>simple</value>
+ <description>
+ Defines authentication used for Oozie HTTP endpoint.
+ Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.token.validity</name>
+ <value>36000</value>
+ <description>
+ Indicates how long (in seconds) an authentication token is valid before it has
+ to be renewed.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.signature.secret.file</name>
+ <value>${user.home}/hadoop-http-auth-signature-secret</value>
+ <description>
+ The signature secret for signing the authentication tokens.
+ The same secret should be used for JT/NN/DN/TT configurations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.cookie.domain</name>
+ <value></value>
+ <description>
+ The domain to use for the HTTP cookie that stores the authentication token.
+ In order to authentiation to work correctly across all Hadoop nodes web-consoles
+ the domain must be correctly set.
+ IMPORTANT: when using IP addresses, browsers ignore cookies with domain settings.
+ For this setting to work properly all nodes in the cluster must be configured
+ to generate URLs with hostname.domain names on it.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description>
+ Indicates if anonymous requests are allowed when using 'simple' authentication.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@LOCALHOST</value>
+ <description>
+ Indicates the Kerberos principal to be used for HTTP endpoint.
+ The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.kerberos.keytab</name>
+ <value>${user.home}/hadoop.keytab</value>
+ <description>
+ Location of the keytab file with the credentials for the principal.
+ Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop.
+ </description>
+ </property>
+
+ <!-- HTTP CORS support -->
+ <property>
+ <name>hadoop.http.cross-origin.enabled</name>
+ <value>false</value>
+ <description>Enable/disable the cross-origin (CORS) filter.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.allowed-origins</name>
+ <value>*</value>
+ <description>Comma separated list of origins that are allowed for web
+ services needing cross-origin (CORS) support. Wildcards (*) and patterns
+ allowed</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.allowed-methods</name>
+ <value>GET,POST,HEAD</value>
+ <description>Comma separated list of methods that are allowed for web
+ services needing cross-origin (CORS) support.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.allowed-headers</name>
+ <value>X-Requested-With,Content-Type,Accept,Origin</value>
+ <description>Comma separated list of headers that are allowed for web
+ services needing cross-origin (CORS) support.</description>
+ </property>
+
+ <property>
+ <name>hadoop.http.cross-origin.max-age</name>
+ <value>1800</value>
+ <description>The number of seconds a pre-flighted request can be cached
+ for web services needing cross-origin (CORS) support.</description>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.methods</name>
+ <value></value>
+ <description>
+ List of fencing methods to use for service fencing. May contain
+ builtin methods (eg shell and sshfence) or user-defined method.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.ssh.connect-timeout</name>
+ <value>30000</value>
+ <description>
+ SSH connection timeout, in milliseconds, to use with the builtin
+ sshfence fencer.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.ssh.private-key-files</name>
+ <value></value>
+ <description>
+ The SSH private key files to use with the builtin sshfence fencer.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.quorum</name>
+ <description>
+ A list of ZooKeeper server addresses, separated by commas, that are
+ to be used by the ZKFailoverController in automatic failover.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.session-timeout.ms</name>
+ <value>5000</value>
+ <description>
+ The session timeout to use when the ZKFC connects to ZooKeeper.
+ Setting this value to a lower value implies that server crashes
+ will be detected more quickly, but risks triggering failover too
+ aggressively in the case of a transient error or network blip.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.parent-znode</name>
+ <value>/hadoop-ha</value>
+ <description>
+ The ZooKeeper znode under which the ZK failover controller stores
+ its information. Note that the nameservice ID is automatically
+ appended to this znode, so it is not normally necessary to
+ configure this, even in a federated environment.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.acl</name>
+ <value>world:anyone:rwcda</value>
+ <description>
+ A comma-separated list of ZooKeeper ACLs to apply to the znodes
+ used by automatic failover. These ACLs are specified in the same
+ format as used by the ZooKeeper CLI.
+
+ If the ACL itself contains secrets, you may instead specify a
+ path to a file, prefixed with the '@' symbol, and the value of
+ this configuration will be loaded from within.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.zookeeper.auth</name>
+ <value></value>
+ <description>
+ A comma-separated list of ZooKeeper authentications to add when
+ connecting to ZooKeeper. These are specified in the same format
+ as used by the "addauth" command in the ZK CLI. It is
+ important that the authentications specified here are sufficient
+ to access znodes with the ACL specified in ha.zookeeper.acl.
+
+ If the auths contain secrets, you may instead specify a
+ path to a file, prefixed with the '@' symbol, and the value of
+ this configuration will be loaded from within.
+ </description>
+ </property>
+
+ <!-- Static Web User Filter properties. -->
+ <property>
+ <name>hadoop.http.staticuser.user</name>
+ <value>dr.who</value>
+ <description>
+ The user name to filter as, on static web filters
+ while rendering content. An example use is the HDFS
+ web UI (user to be used for browsing files).
+ </description>
+ </property>
+
+ <!-- SSLFactory configuration -->
+
+ <property>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ <description>
+ The keystores factory to use for retrieving certificates.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ <description>Whether client certificates are required</description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.hostname.verifier</name>
+ <value>DEFAULT</value>
+ <description>
+ The hostname verifier to provide for HttpsURLConnections.
+ Valid values are: DEFAULT, STRICT, STRICT_IE6, DEFAULT_AND_LOCALHOST and
+ ALLOW_ALL
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ <description>
+ Resource file from which ssl server keystore information will be extracted.
+ This file is looked up in the classpath, typically it should be in Hadoop
+ conf/ directory.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ <description>
+ Resource file from which ssl client keystore information will be extracted
+ This file is looked up in the classpath, typically it should be in Hadoop
+ conf/ directory.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.enabled</name>
+ <value>false</value>
+ <description>
+ Deprecated. Use dfs.http.policy and yarn.http.policy instead.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.enabled.protocols</name>
+ <value>TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2</value>
+ <description>
+ The supported SSL protocols.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.jetty.logs.serve.aliases</name>
+ <value>true</value>
+ <description>
+ Enable/Disable aliases serving from jetty
+ </description>
+ </property>
+
+ <property>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
+ <description>
+ The umask used when creating files and directories.
+ Can be in octal or in symbolic. Examples are:
+ "022" (octal for u=rwx,g=r-x,o=r-x in symbolic),
+ or "u=rwx,g=rwx,o=" (symbolic for 007 in octal).
+ </description>
+ </property>
+
+ <!-- ha properties -->
+
+ <property>
+ <name>ha.health-monitor.connect-retry-interval.ms</name>
+ <value>1000</value>
+ <description>
+ How often to retry connecting to the service.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.health-monitor.check-interval.ms</name>
+ <value>1000</value>
+ <description>
+ How often to check the service.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.health-monitor.sleep-after-disconnect.ms</name>
+ <value>1000</value>
+ <description>
+ How long to sleep after an unexpected RPC error.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.health-monitor.rpc-timeout.ms</name>
+ <value>45000</value>
+ <description>
+ Timeout for the actual monitorHealth() calls.
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.new-active.rpc-timeout.ms</name>
+ <value>60000</value>
+ <description>
+ Timeout that the FC waits for the new active to become active
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.graceful-fence.rpc-timeout.ms</name>
+ <value>5000</value>
+ <description>
+ Timeout that the FC waits for the old active to go to standby
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.graceful-fence.connection.retries</name>
+ <value>1</value>
+ <description>
+ FC connection retries for graceful fencing
+ </description>
+ </property>
+
+ <property>
+ <name>ha.failover-controller.cli-check.rpc-timeout.ms</name>
+ <value>20000</value>
+ <description>
+ Timeout that the CLI (manual) FC waits for monitorHealth, getServiceState
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.client.fallback-to-simple-auth-allowed</name>
+ <value>false</value>
+ <description>
+ When a client is configured to attempt a secure connection, but attempts to
+ connect to an insecure server, that server may instruct the client to
+ switch to SASL SIMPLE (unsecure) authentication. This setting controls
+ whether or not the client will accept this instruction from the server.
+ When false (the default), the client will not allow the fallback to SIMPLE
+ authentication, and will abort the connection.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.client.resolve.remote.symlinks</name>
+ <value>true</value>
+ <description>
+ Whether to resolve symlinks when accessing a remote Hadoop filesystem.
+ Setting this to false causes an exception to be thrown upon encountering
+ a symlink. This setting does not apply to local filesystems, which
+ automatically resolve local symlinks.
+ </description>
+ </property>
+
+ <property>
+ <name>nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ <description>
+ By default, the export can be mounted by any client. The value string
+ contains machine name and access privilege, separated by whitespace
+ characters. The machine name format can be a single host, a Java regular
+ expression, or an IPv4 address. The access privilege uses rw or ro to
+ specify read/write or read-only access of the machines to exports. If the
+ access privilege is not provided, the default is read-only. Entries are separated by ";".
+ For example: "192.168.0.0/22 rw ; host.*\.example\.com ; host1.test.org ro;".
+ Only the NFS gateway needs to restart after this property is updated.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.user.group.static.mapping.overrides</name>
+ <value>dr.who=;</value>
+ <description>
+ Static mapping of user to groups. This will override the groups if
+ available in the system for the specified user. In other words, groups
+ look-up will not happen for these users, instead groups mapped in this
+ configuration will be used.
+ Mapping should be in this format.
+ user1=group1,group2;user2=;user3=group2;
+ Default, "dr.who=;" will consider "dr.who" as user without groups.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.metrics.quantile.enable</name>
+ <value>false</value>
+ <description>
+ Setting this property to true and rpc.metrics.percentiles.intervals
+ to a comma-separated list of the granularity in seconds, the
+ 50/75/90/95/99th percentile latency for rpc queue/processing time in
+ milliseconds are added to rpc metrics.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.metrics.percentiles.intervals</name>
+ <value></value>
+ <description>
+ A comma-separated list of the granularity in seconds for the metrics which
+ describe the 50/75/90/95/99th percentile latency for rpc queue/processing
+ time. The metrics are outputted if rpc.metrics.quantile.enable is set to
+ true.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE</name>
+ <value></value>
+ <description>
+ The prefix for a given crypto codec, contains a comma-separated
+ list of implementation classes for a given crypto codec (eg EXAMPLECIPHERSUITE).
+ The first implementation will be used if available, others are fallbacks.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.codec.classes.aes.ctr.nopadding</name>
+ <value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
+ <description>
+ Comma-separated list of crypto codec implementations for AES/CTR/NoPadding.
+ The first implementation will be used if available, others are fallbacks.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.cipher.suite</name>
+ <value>AES/CTR/NoPadding</value>
+ <description>
+ Cipher suite for crypto codec.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.jce.provider</name>
+ <value></value>
+ <description>
+ The JCE provider name used in CryptoCodec.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.crypto.buffer.size</name>
+ <value>8192</value>
+ <description>
+ The buffer size used by CryptoInputStream and CryptoOutputStream.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.java.secure.random.algorithm</name>
+ <value>SHA1PRNG</value>
+ <description>
+ The java secure random algorithm.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.secure.random.impl</name>
+ <value></value>
+ <description>
+ Implementation of secure random.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.random.device.file.path</name>
+ <value>/dev/urandom</value>
+ <description>
+ OS security random device file path.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.key.provider.path</name>
+ <description>
+ The KeyProvider to use when managing zone keys, and interacting with
+ encryption keys when reading and writing to an encryption zone.
+ For hdfs clients, the provider path will be same as namenode's
+ provider path.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.har.impl.disable.cache</name>
+ <value>true</value>
+ <description>Don't cache 'har' filesystem instances.</description>
+ </property>
+
+ <!--- KMSClientProvider configurations -->
+ <property>
+ <name>hadoop.security.kms.client.authentication.retry-count</name>
+ <value>1</value>
+ <description>
+ Number of time to retry connecting to KMS on authentication failure
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.size</name>
+ <value>500</value>
+ <description>
+ Size of the EncryptedKeyVersion cache Queue for each key
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.low-watermark</name>
+ <value>0.3f</value>
+ <description>
+ If size of the EncryptedKeyVersion cache Queue falls below the
+ low watermark, this cache queue will be scheduled for a refill
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.num.refill.threads</name>
+ <value>2</value>
+ <description>
+ Number of threads to use for refilling depleted EncryptedKeyVersion
+ cache Queues
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.expiry</name>
+ <value>43200000</value>
+ <description>
+ Cache expiry time for a Key, after which the cache Queue for this
+ key will be dropped. Default = 12hrs
+ </description>
+ </property>
+ <property>
+ <name>hadoop.security.kms.client.timeout</name>
+ <value>60</value>
+ <description>
+ Sets value for KMS client connection timeout, and the read timeout
+ to KMS servers.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.kms.client.failover.sleep.base.millis</name>
+ <value>100</value>
+ <description>
+ Expert only. The time to wait, in milliseconds, between failover
+ attempts increases exponentially as a function of the number of
+ attempts made so far, with a random factor of +/- 50%. This option
+ specifies the base value used in the failover calculation. The
+ first failover will retry immediately. The 2nd failover attempt
+ will delay at least hadoop.security.client.failover.sleep.base.millis
+ milliseconds. And so on.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.kms.client.failover.sleep.max.millis</name>
+ <value>2000</value>
+ <description>
+ Expert only. The time to wait, in milliseconds, between failover
+ attempts increases exponentially as a function of the number of
+ attempts made so far, with a random factor of +/- 50%. This option
+ specifies the maximum value to wait between failovers.
+ Specifically, the time between two failover attempts will not
+ exceed +/- 50% of hadoop.security.client.failover.sleep.max.millis
+ milliseconds.
+ </description>
+ </property>
+
+ <property>
+ <name>ipc.server.max.connections</name>
+ <value>0</value>
+ <description>The maximum number of concurrent connections a server is allowed
+ to accept. If this limit is exceeded, incoming connections will first fill
+ the listen queue and then may go to an OS-specific listen overflow queue.
+ The client may fail or timeout, but the server can avoid running out of file
+ descriptors using this feature. 0 means no limit.
+ </description>
+ </property>
+
+
+ <!-- YARN registry -->
+
+ <property>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
+ <description>
+ Is the registry enabled in the YARN Resource Manager?
+
+ If true, the YARN RM will, as needed.
+ create the user and system paths, and purge
+ service records when containers, application attempts
+ and applications complete.
+
+ If false, the paths must be created by other means,
+ and no automatic cleanup of service records will take place.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.root</name>
+ <value>/registry</value>
+ <description>
+ The root zookeeper node for the registry
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.session.timeout.ms</name>
+ <value>60000</value>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.connection.timeout.ms</name>
+ <value>15000</value>
+ <description>
+ Zookeeper connection timeout in milliseconds
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.retry.times</name>
+ <value>5</value>
+ <description>
+ Zookeeper connection retry count before failing
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.retry.interval.ms</name>
+ <value>1000</value>
+ <description>
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.retry.ceiling.ms</name>
+ <value>60000</value>
+ <description>
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff.
+
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ <description>
+ List of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.secure</name>
+ <value>false</value>
+ <description>
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.system.acls</name>
+ <value>sasl:yarn@, sasl:mapred@, sasl:hdfs@</value>
+ <description>
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+
+ These are given full access to all entries.
+
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.kerberos.realm</name>
+ <value></value>
+ <description>
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+
+ If empty, the default realm of the running process
+ is used.
+
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.registry.jaas.context</name>
+ <value>Client</value>
+ <description>
+ Key to define the JAAS context. Used in secure
+ mode
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.shell.missing.defaultFs.warning</name>
+ <value>false</value>
+ <description>
+ Enable hdfs shell commands to display warnings if (fs.defaultFS) property
+ is not set.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.shell.safely.delete.limit.num.files</name>
+ <value>100</value>
+ <description>Used by -safely option of hadoop fs shell -rm command to avoid
+ accidental deletion of large directories. When enabled, the -rm command
+ requires confirmation if the number of files to be deleted is greater than
+ this limit. The default limit is 100 files. The warning is disabled if
+ the limit is 0 or the -safely is not specified in -rm command.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.client.htrace.sampler.classes</name>
+ <value></value>
+ <description>The class names of the HTrace Samplers to use for Hadoop
+ filesystem clients.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.htrace.span.receiver.classes</name>
+ <value></value>
+ <description>The class names of the Span Receivers to use for Hadoop.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.http.logs.enabled</name>
+ <value>true</value>
+ <description>
+ Enable the "/logs" endpoint on all Hadoop daemons, which serves local
+ logs, but may be considered a security risk due to it listing the contents
+ of a directory.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.client.resolve.topology.enabled</name>
+ <value>false</value>
+ <description>Whether the client machine will use the class specified by
+ property net.topology.node.switch.mapping.impl to compute the network
+ distance between itself and remote machines of the FileSystem. Additional
+ properties might need to be configured depending on the class specified
+ in net.topology.node.switch.mapping.impl. For example, if
+ org.apache.hadoop.net.ScriptBasedMapping is used, a valid script file
+ needs to be specified in net.topology.script.file.name.
+ </description>
+ </property>
+
+
+ <!-- Azure Data Lake File System Configurations -->
+
+ <property>
+ <name>fs.adl.impl</name>
+ <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
+ </property>
+
+ <property>
+ <name>fs.AbstractFileSystem.adl.impl</name>
+ <value>org.apache.hadoop.fs.adl.Adl</value>
+ </property>
+
+ <property>
+ <name>adl.feature.ownerandgroup.enableupn</name>
+ <value>false</value>
+ <description>
+ When true : User and Group in FileStatus/AclStatus response is
+ represented as user friendly name as per Azure AD profile.
+
+ When false (default) : User and Group in FileStatus/AclStatus
+ response is represented by the unique identifier from Azure AD
+ profile (Object ID as GUID).
+
+ For optimal performance, false is recommended.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.access.token.provider.type</name>
+ <value>ClientCredential</value>
+ <description>
+ Defines Azure Active Directory OAuth2 access token provider type.
+ Supported types are ClientCredential, RefreshToken, MSI, DeviceCode,
+ and Custom.
+ The ClientCredential type requires property fs.adl.oauth2.client.id,
+ fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
+ The RefreshToken type requires property fs.adl.oauth2.client.id and
+ fs.adl.oauth2.refresh.token.
+ The MSI type reads optional property fs.adl.oauth2.msi.port, if specified.
+ The DeviceCode type requires property
+ fs.adl.oauth2.devicecode.clientapp.id.
+ The Custom type requires property fs.adl.oauth2.access.token.provider.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.client.id</name>
+ <value></value>
+ <description>The OAuth2 client id.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.credential</name>
+ <value></value>
+ <description>The OAuth2 access key.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.refresh.url</name>
+ <value></value>
+ <description>The OAuth2 token endpoint.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.refresh.token</name>
+ <value></value>
+ <description>The OAuth2 refresh token.</description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.access.token.provider</name>
+ <value></value>
+ <description>
+ The class name of the OAuth2 access token provider.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.msi.port</name>
+ <value></value>
+ <description>
+ The localhost port for the MSI token service. This is the port specified
+ when creating the Azure VM. The default, if this setting is not specified,
+ is 50342.
+ Used by MSI token provider.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.adl.oauth2.devicecode.clientapp.id</name>
+ <value></value>
+ <description>
+ The app id of the AAD native app in whose context the auth request
+ should be made.
+ Used by DeviceCode token provider.
+ </description>
+ </property>
+
+ <!-- Azure Data Lake File System Configurations Ends Here-->
+
+ <property>
+ <name>hadoop.caller.context.enabled</name>
+ <value>false</value>
+ <description>When the feature is enabled, additional fields are written into
+ name-node audit log records for auditing coarse granularity operations.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.max.size</name>
+ <value>128</value>
+ <description>The maximum bytes a caller context string can have. If the
+ passed caller context is longer than this maximum bytes, client will
+ truncate it before sending to server. Note that the server may have a
+ different maximum size, and will truncate the caller context to the
+ maximum size it allows.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.signature.max.size</name>
+ <value>40</value>
+ <description>
+ The caller's signature (optional) is for offline validation. If the
+ signature exceeds the maximum allowed bytes in server, the caller context
+ will be abandoned, in which case the caller context will not be recorded
+ in audit logs.
+ </description>
+ </property>
+ <!-- SequenceFile's Sorter properties -->
+ <property>
+ <name>seq.io.sort.mb</name>
+ <value>100</value>
+ <description>
+ The total amount of buffer memory to use while sorting files,
+ while using SequenceFile.Sorter, in megabytes. By default,
+ gives each merge stream 1MB, which should minimize seeks.
+ </description>
+ </property>
+ <property>
+ <name>seq.io.sort.factor</name>
+ <value>100</value>
+ <description>
+ The number of streams to merge at once while sorting
+ files using SequenceFile.Sorter.
+ This determines the number of open file handles.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.address</name>
+ <!--value>127.0.0.1:2181</value-->
+ <description>Host:Port of the ZooKeeper server to be used.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.num-retries</name>
+ <value>1000</value>
+ <description>Number of tries to connect to ZooKeeper.</description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.retry-interval-ms</name>
+ <value>1000</value>
+ <description>Retry interval in milliseconds when connecting to ZooKeeper.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.timeout-ms</name>
+ <value>10000</value>
+ <description>ZooKeeper session timeout in milliseconds. Session expiration
+ is managed by the ZooKeeper cluster itself, not by the client. This value is
+ used by the cluster to determine when the client's session expires.
+ Expirations happens when the cluster does not hear from the client within
+ the specified session timeout period (i.e. no heartbeat).</description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.acl</name>
+ <value>world:anyone:rwcda</value>
+ <description>ACL's to be used for ZooKeeper znodes.</description>
+ </property>
+
+ <property>
+ <name>hadoop.zk.auth</name>
+ <description>
+ Specify the auths to be used for the ACL's specified in hadoop.zk.acl.
+ This takes a comma-separated list of authentication mechanisms, each of the
+ form 'scheme:auth' (the same syntax used for the 'addAuth' command in
+ the ZK CLI).
+ </description>
+ </property>
+ <property>
+ <name>hadoop.treat.subject.external</name>
+ <value>false</value>
+ <description>
+ When creating UGI with UserGroupInformation(Subject), treat the passed
+ subject external if set to true, and assume the owner of the subject
+ should do the credential renewal.
+
+ When true this property will introduce an incompatible change which
+ may require changes in client code. For more details, see the jiras:
+ HADOOP-13805,HADOOP-13558.
+ </description>
+ </property>
+</configuration>
diff --git a/flink-filesystems/pom.xml b/flink-filesystems/pom.xml
index 148eed6..da5c187 100644
--- a/flink-filesystems/pom.xml
+++ b/flink-filesystems/pom.xml
@@ -30,14 +30,18 @@ under the License.
<relativePath>..</relativePath>
</parent>
-
<artifactId>flink-filesystems</artifactId>
<name>flink-filesystems</name>
<packaging>pom</packaging>
+ <properties>
+ <fs.hadoopshaded.version>3.1.0</fs.hadoopshaded.version>
+ </properties>
+
<modules>
<module>flink-hadoop-fs</module>
<module>flink-mapr-fs</module>
+ <module>flink-fs-hadoop-shaded</module>
<module>flink-s3-fs-hadoop</module>
<module>flink-s3-fs-presto</module>
<module>flink-swift-fs-hadoop</module>
diff --git a/tools/maven/suppressions.xml b/tools/maven/suppressions.xml
index 4594d7f..1d969b1 100644
--- a/tools/maven/suppressions.xml
+++ b/tools/maven/suppressions.xml
@@ -51,6 +51,9 @@ under the License.
<suppress
files="org[\\/]apache[\\/]hadoop[\\/]util[\\/]NativeCodeLoader.java"
checks=".*"/>
+ <suppress
+ files="org[\\/]apache[\\/]hadoop[\\/]util[\\/]VersionInfo.java"
+ checks=".*"/>
<!-- This file is generated in the flink-python build process -->
<suppress
files="python-source.zip"