You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@asterixdb.apache.org by "Chris Hillery (Code Review)" <do...@asterix-gerrit.ics.uci.edu> on 2015/07/31 13:06:33 UTC
Change in asterixdb[master]: WIP for Asterix configuration changes.
Chris Hillery has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/335
Change subject: WIP for Asterix configuration changes.
......................................................................
WIP for Asterix configuration changes.
Change-Id: Ie3027c8c839f25ea858790bd3340187f4b11f212
---
M asterix-app/pom.xml
M asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
M asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
M asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
M asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
M asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
M asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
M asterix-server/pom.xml
M pom.xml
9 files changed, 135 insertions(+), 55 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/35/335/1
diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml
index 25f1ad9..8ee5644 100644
--- a/asterix-app/pom.xml
+++ b/asterix-app/pom.xml
@@ -128,6 +128,10 @@
<artifactId>hyracks-client</artifactId>
</dependency>
<dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ </dependency>
+ <dependency>
<groupId>edu.uci.ics.asterix</groupId>
<artifactId>asterix-algebra</artifactId>
<version>0.8.7-SNAPSHOT</version>
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
index 6d7f2a4..ef6267f 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
@@ -41,6 +41,7 @@
import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceRepositoryFactory;
import edu.uci.ics.asterix.transaction.management.service.logging.LogManager;
import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
import edu.uci.ics.hyracks.api.application.INCApplicationContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.IIOManager;
@@ -69,16 +70,6 @@
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactoryProvider;
public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAsterixPropertiesProvider {
-
- private static final AsterixPropertiesAccessor ASTERIX_PROPERTIES_ACCESSOR;
-
- static {
- try {
- ASTERIX_PROPERTIES_ACCESSOR = new AsterixPropertiesAccessor();
- } catch (AsterixException e) {
- throw new ExceptionInInitializerError(e);
- }
- }
private static final int METADATA_IO_DEVICE_ID = 0;
@@ -109,12 +100,22 @@
public AsterixAppRuntimeContext(INCApplicationContext ncApplicationContext) throws AsterixException {
this.ncApplicationContext = ncApplicationContext;
- compilerProperties = new AsterixCompilerProperties(ASTERIX_PROPERTIES_ACCESSOR);
- externalProperties = new AsterixExternalProperties(ASTERIX_PROPERTIES_ACCESSOR);
- metadataProperties = new AsterixMetadataProperties(ASTERIX_PROPERTIES_ACCESSOR);
- storageProperties = new AsterixStorageProperties(ASTERIX_PROPERTIES_ACCESSOR);
- txnProperties = new AsterixTransactionProperties(ASTERIX_PROPERTIES_ACCESSOR);
- feedProperties = new AsterixFeedProperties(ASTERIX_PROPERTIES_ACCESSOR);
+ // Determine whether to use old-style asterix-configuration.xml or new-style configuration.
+ // QQQ strip this out eventually
+ AsterixPropertiesAccessor propertiesAccessor;
+ IApplicationConfig cfg = ncApplicationContext.getAppConfig();
+ // QQQ this is NOT a good way to determine whether the config is valid
+ if (cfg.getString("cc", "cluster.address") != null) {
+ propertiesAccessor = new AsterixPropertiesAccessor(cfg);
+ } else {
+ propertiesAccessor = new AsterixPropertiesAccessor();;
+ }
+ compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
+ externalProperties = new AsterixExternalProperties(propertiesAccessor);
+ metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
+ storageProperties = new AsterixStorageProperties(propertiesAccessor);
+ txnProperties = new AsterixTransactionProperties(propertiesAccessor);
+ feedProperties = new AsterixFeedProperties(propertiesAccessor);
}
public void initialize() throws IOException, ACIDException, AsterixException {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 956b447..399a993 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -21,6 +21,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
index fc8b3df..d800009 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
@@ -19,6 +19,7 @@
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -35,18 +36,24 @@
import edu.uci.ics.asterix.common.configuration.Store;
import edu.uci.ics.asterix.common.configuration.TransactionLogDir;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
public class AsterixPropertiesAccessor {
private static final Logger LOGGER = Logger.getLogger(AsterixPropertiesAccessor.class.getName());
private final String instanceName;
private final String metadataNodeName;
- private final Set<String> nodeNames;
- private final Map<String, String[]> stores;
- private final Map<String, String> coredumpConfig;
+ private final Set<String> nodeNames = new HashSet<>();
+ private final Map<String, String[]> stores = new HashMap<>();
+ private final Map<String, String> coredumpConfig = new HashMap<>();
+ private final Map<String, String> transactionLogDirs = new HashMap<>();
private final Map<String, Property> asterixConfigurationParams;
- private final Map<String, String> transactionLogDirs;
+ private final IApplicationConfig cfg;
+ /**
+ * Constructor which reads asterix-configuration.xml, the old way.
+ * @throws AsterixException
+ */
public AsterixPropertiesAccessor() throws AsterixException {
String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY);
if (fileName == null) {
@@ -72,9 +79,7 @@
}
instanceName = asterixConfiguration.getInstanceName();
metadataNodeName = asterixConfiguration.getMetadataNode();
- stores = new HashMap<String, String[]>();
List<Store> configuredStores = asterixConfiguration.getStore();
- nodeNames = new HashSet<String>();
for (Store store : configuredStores) {
String trimmedStoreDirs = store.getStoreDirs().trim();
stores.put(store.getNcId(), trimmedStoreDirs.split(","));
@@ -84,15 +89,50 @@
for (Property p : asterixConfiguration.getProperty()) {
asterixConfigurationParams.put(p.getName(), p);
}
- coredumpConfig = new HashMap<String, String>();
for (Coredump cd : asterixConfiguration.getCoredump()) {
coredumpConfig.put(cd.getNcId(), cd.getCoredumpPath());
}
- transactionLogDirs = new HashMap<String, String>();
for (TransactionLogDir txnLogDir : asterixConfiguration.getTransactionLogDir()) {
transactionLogDirs.put(txnLogDir.getNcId(), txnLogDir.getTxnLogDirPath());
}
+ cfg = null;
+ }
+
+ /**
+ * Constructor which wraps an IApplicationConfig.
+ */
+ public AsterixPropertiesAccessor (IApplicationConfig cfg) {
+ this.cfg = cfg;
+ instanceName = cfg.getString("asterix", "instance", "DEFAULT_INSTANCE");
+ String mdNode = null;
+ for (String section : cfg.getSections()) {
+ if (!section.startsWith("nc/")) {
+ continue;
+ }
+ String ncId = section.substring(3);
+ nodeNames.add(ncId);
+
+ if (mdNode == null) {
+ // Default is first node == metadata node
+ mdNode = ncId;
+ }
+ if (cfg.getString(section, "metadata.port") != null) {
+ // QQQ But we don't actually *honor* metadata.port yet!
+ mdNode = ncId;
+ }
+
+ // QQQ Default values? Should they be specified here? Or should there
+ // be a default.ini? They can't be inserted by TriggerNCWork except
+ // possibly for hyracks-specified values. Certainly wherever they are,
+ // they should be platform-dependent.
+ stores.put(ncId, cfg.getString(section, "iodevices", "/var/lib/asterixdb/data").split(","));
+ coredumpConfig.put(ncId, cfg.getString(section, "coredumpdir", "/var/lib/asterixdb/coredump"));
+ transactionLogDirs.put(ncId, cfg.getString(section, "txnlogdir", "/var/lib/asterixdb/txn-log"));
+ }
+
+ metadataNodeName = mdNode;
+ asterixConfigurationParams = null;
}
public String getMetadataNodeName() {
@@ -138,24 +178,33 @@
}
public <T> T getProperty(String property, T defaultValue, IPropertyInterpreter<T> interpreter) {
- Property p = asterixConfigurationParams.get(property);
- if (p == null) {
+ String value;
+ Property p = null;
+ if (asterixConfigurationParams != null) {
+ p = asterixConfigurationParams.get(property);
+ value = (p == null) ? null : p.getValue();
+ } else {
+ value = cfg.getString("asterix", property);
+ }
+ if (value == null) {
return defaultValue;
}
-
try {
- return interpreter.interpret(p);
+ return interpreter.interpret(value);
} catch (IllegalArgumentException e) {
- logConfigurationError(p, defaultValue);
+ if (LOGGER.isLoggable(Level.SEVERE)) {
+ StringBuilder msg = new StringBuilder("Invalid property value '" + value + "' for property '" + property + "'.\n");
+ if (p != null) {
+ msg.append("See the description: \n" + p.getDescription() + "\n");
+ }
+ msg.append("Default = " + defaultValue);
+ LOGGER.severe(msg.toString());
+ }
throw e;
}
}
private <T> void logConfigurationError(Property p, T defaultValue) {
- if (LOGGER.isLoggable(Level.SEVERE)) {
- LOGGER.severe("Invalid property value '" + p.getValue() + "' for property '" + p.getName()
- + "'.\n See the description: \n" + p.getDescription() + "\nDefault = " + defaultValue);
- }
}
public String getInstanceName() {
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
index 67acf4b..c4acdd9 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
@@ -17,5 +17,5 @@
import edu.uci.ics.asterix.common.configuration.Property;
public interface IPropertyInterpreter<T> {
- public T interpret(Property p) throws IllegalArgumentException;
+ public T interpret(String s) throws IllegalArgumentException;
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
index 465d7c5..1c7256b 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
@@ -22,11 +22,10 @@
public static IPropertyInterpreter<Integer> getIntegerPropertyInterpreter() {
return new IPropertyInterpreter<Integer>() {
-
@Override
- public Integer interpret(Property p) throws IllegalArgumentException {
+ public Integer interpret(String s) throws IllegalArgumentException {
try {
- return Integer.parseInt(p.getValue());
+ return Integer.parseInt(s);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(e);
}
@@ -36,20 +35,19 @@
public static IPropertyInterpreter<Boolean> getBooleanPropertyInterpreter() {
return new IPropertyInterpreter<Boolean>() {
-
- public Boolean interpret(Property p) throws IllegalArgumentException {
- return Boolean.parseBoolean(p.getValue());
+ @Override
+ public Boolean interpret(String s) throws IllegalArgumentException {
+ return Boolean.parseBoolean(s);
}
};
}
public static IPropertyInterpreter<Long> getLongPropertyInterpreter() {
return new IPropertyInterpreter<Long>() {
-
@Override
- public Long interpret(Property p) throws IllegalArgumentException {
+ public Long interpret(String s) throws IllegalArgumentException {
try {
- return Long.parseLong(p.getValue());
+ return Long.parseLong(s);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(e);
}
@@ -59,31 +57,28 @@
public static IPropertyInterpreter<Level> getLevelPropertyInterpreter() {
return new IPropertyInterpreter<Level>() {
-
@Override
- public Level interpret(Property p) throws IllegalArgumentException {
- return Level.parse(p.getValue());
+ public Level interpret(String s) throws IllegalArgumentException {
+ return Level.parse(s);
}
};
}
public static IPropertyInterpreter<String> getStringPropertyInterpreter() {
return new IPropertyInterpreter<String>() {
-
@Override
- public String interpret(Property p) throws IllegalArgumentException {
- return p.getValue();
+ public String interpret(String s) throws IllegalArgumentException {
+ return s;
}
};
}
public static IPropertyInterpreter<Double> getDoublePropertyInterpreter() {
return new IPropertyInterpreter<Double>() {
-
@Override
- public Double interpret(Property p) throws IllegalArgumentException {
+ public Double interpret(String s) throws IllegalArgumentException {
try {
- return Double.parseDouble(p.getValue());
+ return Double.parseDouble(s);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(e);
}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
index fece403..2ff248b 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
@@ -27,6 +27,7 @@
import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -52,10 +53,21 @@
private IHyracksClientConnection hcc;
public static void initialize(ICCApplicationContext ccAppCtx, IHyracksClientConnection hcc) throws AsterixException {
- if (INSTANCE == null) {
- INSTANCE = new AsterixAppContextInfo(ccAppCtx, hcc);
+ if (INSTANCE != null) {
+ return;
}
- AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
+ INSTANCE = new AsterixAppContextInfo(ccAppCtx, hcc);
+
+ // Determine whether to use old-style asterix-configuration.xml or new-style configuration.
+ // QQQ strip this out eventually
+ AsterixPropertiesAccessor propertiesAccessor;
+ IApplicationConfig cfg = ccAppCtx.getAppConfig();
+ // QQQ this is NOT a good way to determine whether the config is valid
+ if (cfg.getString("cc", "cluster.address") != null) {
+ propertiesAccessor = new AsterixPropertiesAccessor(cfg);
+ } else {
+ propertiesAccessor = new AsterixPropertiesAccessor();
+ }
INSTANCE.compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
INSTANCE.externalProperties = new AsterixExternalProperties(propertiesAccessor);
INSTANCE.metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
diff --git a/asterix-server/pom.xml b/asterix-server/pom.xml
index 203e5d2..a542212 100644
--- a/asterix-server/pom.xml
+++ b/asterix-server/pom.xml
@@ -65,6 +65,13 @@
<commandLineArgument>edu.uci.ics.asterix.hyracks.bootstrap.NCApplicationEntryPoint</commandLineArgument>
</commandLineArguments>
</program>
+ <program>
+ <platforms>
+ <platform>unix</platform>
+ </platforms>
+ <name>asterixncservice</name>
+ <mainClass>edu.uci.ics.hyracks.control.nc.service.NCService</mainClass>
+ </program>
</programs>
<daemons>
<daemon>
@@ -134,6 +141,12 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-nc-service</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>edu.uci.ics.asterix</groupId>
<artifactId>asterix-app</artifactId>
<version>0.8.7-SNAPSHOT</version>
diff --git a/pom.xml b/pom.xml
index 6f4217e..caa7d3b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -357,6 +357,11 @@
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-nc-service</artifactId>
+ <version>${hyracks.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-server</artifactId>
<version>${hyracks.version}</version>
</dependency>
--
To view, visit https://asterix-gerrit.ics.uci.edu/335
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie3027c8c839f25ea858790bd3340187f4b11f212
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Chris Hillery <ce...@lambda.nu>