You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ka...@apache.org on 2012/11/10 10:49:43 UTC
svn commit: r1407739 - in /oozie/branches/hcat-intre:
core/src/main/java/org/apache/oozie/util/HCatURI.java
core/src/test/java/org/apache/oozie/util/TestHCatURI.java release-log.txt
Author: kamrul
Date: Sat Nov 10 09:49:42 2012
New Revision: 1407739
URL: http://svn.apache.org/viewvc?rev=1407739&view=rev
Log:
OOZIE-1059 Add static method to create URI String in HCatURI(ryota via mohammad)
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java
oozie/branches/hcat-intre/release-log.txt
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java?rev=1407739&r1=1407738&r2=1407739&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/util/HCatURI.java Sat Nov 10 09:49:42 2012
@@ -20,18 +20,25 @@ package org.apache.oozie.util;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
+import java.util.Map.Entry;
+
import org.apache.hadoop.conf.Configuration;
+/**
+ * Utility class to parse HCatalog URI
+ */
public class HCatURI {
public static final String PREFIX_HCAT = "oozie.service.MetaAccessorService.hcat";
public static final String DEFAULT_SERVER = PREFIX_HCAT + ".server";
public static final String DEFAULT_DB = PREFIX_HCAT + ".db";
public static final String DEFAULT_TABLE = PREFIX_HCAT + ".table";
- public static final String PARTITION_SEPARATOR = ";";
+ public static final String PARTITION_SEPARATOR = "&";
public static final String PARTITION_KEYVAL_SEPARATOR = "=";
public static final String PATH_SEPARATOR = "/";
+ public static final String PARTITION_PREFIX = "?";
private URI uri;
private String server;
@@ -39,10 +46,21 @@ public class HCatURI {
private String table;
private HashMap<String, String> partitions;
+ /**
+ * Constructor using given configuration
+ * @param s HCat URI String
+ * @param conf Configuration
+ * @throws URISyntaxException
+ */
public HCatURI(String s, Configuration conf) throws URISyntaxException {
parse(s, conf);
}
+ /**
+ * Constructor using default configuration
+ * @param s HCat URI String
+ * @throws URISyntaxException
+ */
public HCatURI(String s) throws URISyntaxException {
this(s, null);
}
@@ -106,50 +124,144 @@ public class HCatURI {
}
}
+ /**
+ * @return server name
+ */
public String getServer() {
return server;
}
+ /**
+ * @param server name to set
+ */
public void setServer(String server) {
this.server = server;
}
+ /**
+ * @return DB name
+ */
public String getDb() {
return db;
}
+ /**
+ * @param DB name to set
+ */
public void setDb(String db) {
this.db = db;
}
+ /**
+ * @return table name
+ */
public String getTable() {
return table;
}
+ /**
+ * @param table name to set
+ */
public void setTable(String table) {
this.table = table;
}
+ /**
+ * @return partitions map
+ */
public HashMap<String, String> getPartitionMap() {
return partitions;
}
+ /**
+ * @param partitions map to set
+ */
public void setPartitionMap(HashMap<String, String> partitions) {
this.partitions = partitions;
}
+ /**
+ * @param key partition key
+ * @return partition value
+ */
public String getPartitionValue(String key) {
return partitions.get(key);
}
- public String setPartition(String key, String value) {
- return partitions.put(key, value);
+ /**
+ * @param key partition key to set
+ * @param value partition value to set
+ */
+ public void setPartition(String key, String value) {
+ partitions.put(key, value);
}
+ /**
+ * @param key partition key
+ * @return if partitions map includes the key or not
+ */
public boolean hasPartition(String key) {
return partitions.containsKey(key);
}
+ /**
+ * static method to create HCatalog URI String
+ *
+ * @param server
+ * @param db
+ * @param table
+ * @param partitions Partition Map
+ * @return
+ */
+ public static String getHCatURI(String server, String db, String table, Map<String, String> partitions) {
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("hcat://");
+ sb.append(server);
+ sb.append(PATH_SEPARATOR);
+ sb.append(db);
+ sb.append(PATH_SEPARATOR);
+ sb.append(table);
+ sb.append(PATH_SEPARATOR);
+ boolean first = true;
+ for (Entry<String, String> entry : partitions.entrySet()) {
+ if (first) {
+ sb.append(PARTITION_PREFIX);
+ }
+ else {
+ sb.append(PARTITION_SEPARATOR);
+ }
+ sb.append(entry.getKey());
+ sb.append(PARTITION_KEYVAL_SEPARATOR);
+ sb.append(entry.getValue());
+ first = false;
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ HCatURI uri = (HCatURI) obj;
+ boolean equals = true;
+ HashMap<String, String> p = this.getPartitionMap();
+ if (this.server.equals(uri.getServer()) && this.db.equals(uri.getDb()) && this.table.equals(uri.getTable())
+ && p.size() == uri.getPartitionMap().size()) {
+ Iterator<Map.Entry<String, String>> it1 = uri.getPartitionMap().entrySet().iterator();
+ while (it1.hasNext()) {
+ Map.Entry<String, String> entry = it1.next();
+ String key = entry.getKey();
+ if (!(p.containsKey(key) && p.get(key).equals(entry.getValue()))) {
+ equals = false;
+ }
+ }
+ }
+ else {
+ equals = false;
+ }
+ return equals;
+ }
+
+ @Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("URI: ");
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java?rev=1407739&r1=1407738&r2=1407739&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/util/TestHCatURI.java Sat Nov 10 09:49:42 2012
@@ -19,6 +19,9 @@ package org.apache.oozie.util;
import static org.junit.Assert.*;
import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.util.HCatURI;
@@ -27,137 +30,206 @@ public class TestHCatURI {
@Test
public void testHCatURIParseValidURI() {
- String input = "hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=12;region=us";
+ String input = "hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=12®ion=us";
HCatURI uri = null;
- try{
- uri= new HCatURI(input);
- }catch (Exception ex){
+ try {
+ uri = new HCatURI(input);
+ }
+ catch (Exception ex) {
System.err.print(ex.getMessage());
}
- assertEquals(uri.getServer(),"hcat.yahoo.com:5080");
- assertEquals(uri.getDb(),"mydb");
- assertEquals(uri.getTable(),"clicks");
- assertEquals(uri.getPartitionValue("datastamp"),"12");
- assertEquals(uri.getPartitionValue("region"),"us");
+ assertEquals(uri.getServer(), "hcat.yahoo.com:5080");
+ assertEquals(uri.getDb(), "mydb");
+ assertEquals(uri.getTable(), "clicks");
+ assertEquals(uri.getPartitionValue("datastamp"), "12");
+ assertEquals(uri.getPartitionValue("region"), "us");
}
@Test
public void testHCatURIParseWithDefaultServer() {
- String input = "hcat:///mydb/clicks/?datastamp=12;region=us";
+ String input = "hcat:///mydb/clicks/?datastamp=12®ion=us";
Configuration conf = new Configuration(false);
conf.set("oozie.service.MetaAccessorService.hcat.server", "hcat.yahoo.com:5080");
conf.set("oozie.service.MetaAccessorService.hcat.db", "mydb");
conf.set("oozie.service.MetaAccessorService.hcat.table", "clicks");
HCatURI uri = null;
- try{
- uri= new HCatURI(input,conf);
- }catch (Exception ex){
+ try {
+ uri = new HCatURI(input, conf);
+ }
+ catch (Exception ex) {
System.err.println(ex.getMessage());
}
- assertEquals(uri.getServer(),"hcat.yahoo.com:5080");
- assertEquals(uri.getDb(),"mydb");
- assertEquals(uri.getTable(),"clicks");
- assertEquals(uri.getPartitionValue("datastamp"),"12");
- assertEquals(uri.getPartitionValue("region"),"us");
+ assertEquals(uri.getServer(), "hcat.yahoo.com:5080");
+ assertEquals(uri.getDb(), "mydb");
+ assertEquals(uri.getTable(), "clicks");
+ assertEquals(uri.getPartitionValue("datastamp"), "12");
+ assertEquals(uri.getPartitionValue("region"), "us");
}
@Test
public void testHCatURIParseWithDefaultDB() {
- String input = "hcat://hcat.yahoo.com:5080//clicks/?datastamp=12;region=us";
+ String input = "hcat://hcat.yahoo.com:5080//clicks/?datastamp=12®ion=us";
Configuration conf = new Configuration(false);
conf.set("oozie.service.MetaAccessorService.hcat.server", "hcat.yahoo.com:5080");
conf.set("oozie.service.MetaAccessorService.hcat.db", "mydb");
conf.set("oozie.service.MetaAccessorService.hcat.table", "clicks");
HCatURI uri = null;
- try{
- uri= new HCatURI(input,conf);
- }catch (Exception ex){
+ try {
+ uri = new HCatURI(input, conf);
+ }
+ catch (Exception ex) {
System.err.println(ex.getMessage());
}
- assertEquals(uri.getServer(),"hcat.yahoo.com:5080");
- assertEquals(uri.getDb(),"mydb");
- assertEquals(uri.getTable(),"clicks");
- assertEquals(uri.getPartitionValue("datastamp"),"12");
- assertEquals(uri.getPartitionValue("region"),"us");
+ assertEquals(uri.getServer(), "hcat.yahoo.com:5080");
+ assertEquals(uri.getDb(), "mydb");
+ assertEquals(uri.getTable(), "clicks");
+ assertEquals(uri.getPartitionValue("datastamp"), "12");
+ assertEquals(uri.getPartitionValue("region"), "us");
}
@Test
public void testHCatURIParseWithDefaultTable() {
- String input = "hcat://hcat.yahoo.com:5080/mydb//?datastamp=12;region=us";
+ String input = "hcat://hcat.yahoo.com:5080/mydb//?datastamp=12®ion=us";
Configuration conf = new Configuration(false);
conf.set("oozie.service.MetaAccessorService.hcat.server", "hcat.yahoo.com:5080");
conf.set("oozie.service.MetaAccessorService.hcat.db", "mydb");
conf.set("oozie.service.MetaAccessorService.hcat.table", "clicks");
HCatURI uri = null;
- try{
- uri= new HCatURI(input,conf);
- }catch (Exception ex){
+ try {
+ uri = new HCatURI(input, conf);
+ }
+ catch (Exception ex) {
System.err.println(ex.getMessage());
}
- assertEquals(uri.getServer(),"hcat.yahoo.com:5080");
- assertEquals(uri.getDb(),"mydb");
- assertEquals(uri.getTable(),"clicks");
- assertEquals(uri.getPartitionValue("datastamp"),"12");
- assertEquals(uri.getPartitionValue("region"),"us");
+ assertEquals(uri.getServer(), "hcat.yahoo.com:5080");
+ assertEquals(uri.getDb(), "mydb");
+ assertEquals(uri.getTable(), "clicks");
+ assertEquals(uri.getPartitionValue("datastamp"), "12");
+ assertEquals(uri.getPartitionValue("region"), "us");
}
-
@Test
public void testHCatURIParseWithAllDefault() {
- String input = "hcat://///?datastamp=12;region=us";
+ String input = "hcat://///?datastamp=12®ion=us";
Configuration conf = new Configuration(false);
conf.set("oozie.service.MetaAccessorService.hcat.server", "hcat.yahoo.com:5080");
conf.set("oozie.service.MetaAccessorService.hcat.db", "mydb");
conf.set("oozie.service.MetaAccessorService.hcat.table", "clicks");
HCatURI uri = null;
- try{
- uri= new HCatURI(input,conf);
- }catch (Exception ex){
+ try {
+ uri = new HCatURI(input, conf);
+ }
+ catch (Exception ex) {
System.err.println(ex.getMessage());
}
- assertEquals(uri.getServer(),"hcat.yahoo.com:5080");
- assertEquals(uri.getDb(),"mydb");
- assertEquals(uri.getTable(),"clicks");
- assertEquals(uri.getPartitionValue("datastamp"),"12");
- assertEquals(uri.getPartitionValue("region"),"us");
+ assertEquals(uri.getServer(), "hcat.yahoo.com:5080");
+ assertEquals(uri.getDb(), "mydb");
+ assertEquals(uri.getTable(), "clicks");
+ assertEquals(uri.getPartitionValue("datastamp"), "12");
+ assertEquals(uri.getPartitionValue("region"), "us");
}
@Test(expected = URISyntaxException.class)
- public void testHCatURIParseInvalidURI() throws Exception{
- String input = "hcat://hcat.yahoo.com:5080/ mydb/clicks/?datastamp=12;region=us";
+ public void testHCatURIParseInvalidURI() throws Exception {
+ String input = "hcat://hcat.yahoo.com:5080/ mydb/clicks/?datastamp=12®ion=us";
HCatURI uri = new HCatURI(input);
}
@Test(expected = URISyntaxException.class)
- public void testHCatURIParseInvalidPartition() throws Exception{
+ public void testHCatURIParseInvalidPartition() throws Exception {
String input = "hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp";
HCatURI uri = new HCatURI(input);
}
@Test(expected = URISyntaxException.class)
- public void testHCatURIParseServerMissing() throws Exception{
+ public void testHCatURIParseServerMissing() throws Exception {
String input = "hcat:///mydb/clicks/?datastamp=12;region=us";
HCatURI uri = new HCatURI(input);
}
@Test(expected = URISyntaxException.class)
- public void testHCatURIParseDBMissing() throws Exception{
+ public void testHCatURIParseDBMissing() throws Exception {
String input = "hcat://hcat.yahoo.com:5080//clicks/?datastamp=12;region=us";
HCatURI uri = new HCatURI(input);
}
@Test(expected = URISyntaxException.class)
- public void testHCatURIParseTableMissing() throws Exception{
+ public void testHCatURIParseTableMissing() throws Exception {
String input = "hcat://hcat.yahoo.com:5080/mydb//?datastamp=12;region=us";
HCatURI uri = new HCatURI(input);
}
+
+ @Test
+ public void testGetHCatUri() {
+ Map<String, String> partitions = new HashMap<String, String>();
+ partitions.put("datastamp", "12");
+ partitions.put("region", "us");
+ String hcatUri = HCatURI.getHCatURI("hcat.yahoo.com:5080", "mydb", "clicks", partitions);
+
+ HCatURI uri1 = null;
+ HCatURI uri2 = null;
+ try {
+ uri1 = new HCatURI(hcatUri);
+ uri2 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=12®ion=us");
+ }
+ catch (URISyntaxException e) {
+ fail(e.getMessage());
+ }
+ assertTrue(uri1.equals(uri2));
+ }
+
+ @Test
+ public void testEqualsPositive() {
+ HCatURI uri1 = null;
+ HCatURI uri2 = null;
+ HCatURI uri3 = null;
+ try {
+ uri1 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=12®ion=us×tamp=1201");
+ uri2 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=12®ion=us×tamp=1201");
+ uri3 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks/?region=us×tamp=1201&datastamp=12");
+ }
+ catch (URISyntaxException e) {
+ fail(e.getMessage());
+ }
+ assertTrue(uri1.equals(uri2));
+ assertTrue(uri2.equals(uri1));
+ assertTrue(uri1.equals(uri3));
+ assertTrue(uri3.equals(uri1));
+ }
+
+ @Test
+ public void testEqualsNegative() {
+ HCatURI uri1 = null;
+ HCatURI uri2 = null;
+ HCatURI uri3 = null;
+ HCatURI uri4 = null;
+ HCatURI uri5 = null;
+ try {
+ uri1 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks/?datastamp=12®ion=us×tamp=1201");
+ uri2 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb2/clicks/?region=us×tamp=1201&datastamp=12");
+ uri3 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks2/?region=us×tamp=1201&datastamp=12");
+ uri4 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks/?region=uk×tamp=1201&datastamp=12");
+ uri5 = new HCatURI("hcat://hcat.yahoo.com:5080/mydb/clicks/?region=us×tamp=1201");
+ }
+ catch (URISyntaxException e) {
+ fail(e.getMessage());
+ }
+ assertFalse(uri1.equals(uri2));
+ assertFalse(uri2.equals(uri1));
+ assertFalse(uri1.equals(uri3));
+ assertFalse(uri3.equals(uri1));
+ assertFalse(uri1.equals(uri4));
+ assertFalse(uri4.equals(uri1));
+ assertFalse(uri1.equals(uri5));
+ assertFalse(uri5.equals(uri1));
+ }
}
Modified: oozie/branches/hcat-intre/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1407739&r1=1407738&r2=1407739&view=diff
==============================================================================
--- oozie/branches/hcat-intre/release-log.txt (original)
+++ oozie/branches/hcat-intre/release-log.txt Sat Nov 10 09:49:42 2012
@@ -1,5 +1,6 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1059 Add static method to create URI String in HCatURI(ryota via mohammad)
OOZIE-1039 Implement the Missing Dependency structure for HCat partitions (mona via mohammad)
OOZIE-1028 Add EL function to allow date ranges to be used for dataset ranges (rkanter via tucu)
OOZIE-1045 Parameterize <unresolved-instances> tag currently hardcoded (egashira via mona)