You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/09/24 20:28:38 UTC
[2/2] git commit: YARN-2102. Added the concept of a Timeline Domain
to handle read/write ACLs on Timeline service event data. Contributed by
Zhijie Shen.
YARN-2102. Added the concept of a Timeline Domain to handle read/write ACLs on Timeline service event data. Contributed by Zhijie Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d78b452a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d78b452a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d78b452a
Branch: refs/heads/trunk
Commit: d78b452a4f413c6931a494c33df0666ce9b44973
Parents: 073bbd8
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Sep 24 11:27:53 2014 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 24 11:27:53 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/records/timeline/TimelineDomain.java | 194 ++++++++++++++++
.../api/records/timeline/TimelineDomains.java | 86 +++++++
.../hadoop/yarn/client/api/TimelineClient.java | 17 ++
.../client/api/impl/TimelineClientImpl.java | 116 +++++++---
.../records/timeline/TestTimelineRecords.java | 38 +++
.../client/api/impl/TestTimelineClient.java | 85 ++++++-
.../server/timeline/LeveldbTimelineStore.java | 216 ++++++++++++++++-
.../server/timeline/MemoryTimelineStore.java | 99 ++++++++
.../server/timeline/TimelineDataManager.java | 74 ++++++
.../yarn/server/timeline/TimelineReader.java | 23 ++
.../yarn/server/timeline/TimelineWriter.java | 20 +-
.../timeline/security/TimelineACLsManager.java | 26 +++
.../timeline/webapp/TimelineWebServices.java | 100 ++++++++
.../timeline/TestLeveldbTimelineStore.java | 19 +-
.../timeline/TestMemoryTimelineStore.java | 15 +-
.../server/timeline/TimelineStoreTestUtils.java | 96 +++++++-
.../security/TestTimelineACLsManager.java | 63 ++++-
.../webapp/TestTimelineWebServices.java | 231 ++++++++++++++++++-
.../webapp/TestTimelineWebServicesWithSSL.java | 5 +-
20 files changed, 1449 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 59287c2..016ac15 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -94,6 +94,9 @@ Release 2.6.0 - UNRELEASED
YARN-2569. Added the log handling APIs for the long running services. (Xuan
Gong via zjshen)
+ YARN-2102. Added the concept of a Timeline Domain to handle read/write ACLs
+ on Timeline service event data. (Zhijie Shen via vinodkv)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomain.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomain.java
new file mode 100644
index 0000000..f8a7e60
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomain.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timeline;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p>
+ * This class contains the information about a timeline domain, which is used
+ * to a user to host a number of timeline entities, isolating them from others'.
+ * The user can also define the reader and writer users/groups for the the
+ * domain, which is used to control the access to its entities.
+ * </p>
+ *
+ * <p>
+ * The reader and writer users/groups pattern that the user can supply is the
+ * same as what <code>AccessControlList</code> takes.
+ * </p>
+ *
+ */
+@XmlRootElement(name = "domain")
+@XmlAccessorType(XmlAccessType.NONE)
+@Public
+@Unstable
+public class TimelineDomain {
+
+ private String id;
+ private String description;
+ private String owner;
+ private String readers;
+ private String writers;
+ private Long createdTime;
+ private Long modifiedTime;
+
+ public TimelineDomain() {
+ }
+
+ /**
+ * Get the domain ID
+ *
+ * @return the domain ID
+ */
+ @XmlElement(name = "id")
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * Set the domain ID
+ *
+ * @param id the domain ID
+ */
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Get the domain description
+ *
+ * @return the domain description
+ */
+ @XmlElement(name = "description")
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * Set the domain description
+ *
+ * @param description the domain description
+ */
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Get the domain owner
+ *
+ * @return the domain owner
+ */
+ @XmlElement(name = "owner")
+ public String getOwner() {
+ return owner;
+ }
+
+ /**
+ * Set the domain owner. The user doesn't need to set it, which will
+ * automatically set to the user who puts the domain.
+ *
+ * @param owner the domain owner
+ */
+ public void setOwner(String owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Get the reader (and/or reader group) list string
+ *
+ * @return the reader (and/or reader group) list string
+ */
+ @XmlElement(name = "readers")
+ public String getReaders() {
+ return readers;
+ }
+
+ /**
+ * Set the reader (and/or reader group) list string
+ *
+ * @param readers the reader (and/or reader group) list string
+ */
+ public void setReaders(String readers) {
+ this.readers = readers;
+ }
+
+ /**
+ * Get the writer (and/or writer group) list string
+ *
+ * @return the writer (and/or writer group) list string
+ */
+ @XmlElement(name = "writers")
+ public String getWriters() {
+ return writers;
+ }
+
+ /**
+ * Set the writer (and/or writer group) list string
+ *
+ * @param writers the writer (and/or writer group) list string
+ */
+ public void setWriters(String writers) {
+ this.writers = writers;
+ }
+
+ /**
+ * Get the created time of the domain
+ *
+ * @return the created time of the domain
+ */
+ @XmlElement(name = "createdtime")
+ public Long getCreatedTime() {
+ return createdTime;
+ }
+
+ /**
+ * Set the created time of the domain
+ *
+ * @param createdTime the created time of the domain
+ */
+ public void setCreatedTime(Long createdTime) {
+ this.createdTime = createdTime;
+ }
+
+ /**
+ * Get the modified time of the domain
+ *
+ * @return the modified time of the domain
+ */
+ @XmlElement(name = "modifiedtime")
+ public Long getModifiedTime() {
+ return modifiedTime;
+ }
+
+ /**
+ * Set the modified time of the domain
+ *
+ * @param modifiedTime the modified time of the domain
+ */
+ public void setModifiedTime(Long modifiedTime) {
+ this.modifiedTime = modifiedTime;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomains.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomains.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomains.java
new file mode 100644
index 0000000..0da1a70
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDomains.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timeline;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * The class that hosts a list of timeline domains.
+ */
+@XmlRootElement(name = "domains")
+@XmlAccessorType(XmlAccessType.NONE)
+@Public
+@Unstable
+public class TimelineDomains {
+
+ private List<TimelineDomain> domains = new ArrayList<TimelineDomain>();
+
+ public TimelineDomains() {
+ }
+
+ /**
+ * Get a list of domains
+ *
+ * @return a list of domains
+ */
+ @XmlElement(name = "domains")
+ public List<TimelineDomain> getDomains() {
+ return domains;
+ }
+
+ /**
+ * Add a single domain into the existing domain list
+ *
+ * @param domain
+ * a single domain
+ */
+ public void addDomain(TimelineDomain domain) {
+ domains.add(domain);
+ }
+
+ /**
+ * All a list of domains into the existing domain list
+ *
+ * @param domains
+ * a list of domains
+ */
+ public void addDomains(List<TimelineDomain> domains) {
+ this.domains.addAll(domains);
+ }
+
+ /**
+ * Set the domain list to the given list of domains
+ *
+ * @param domains
+ * a list of domains
+ */
+ public void setDomains(List<TimelineDomain> domains) {
+ this.domains = domains;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index de1d3e2..27987cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -69,6 +70,22 @@ public abstract class TimelineClient extends AbstractService {
/**
* <p>
+ * Send the information of a domain to the timeline server. It is a
+ * blocking API. The method will not return until it gets the response from
+ * the timeline server.
+ * </p>
+ *
+ * @param domain
+ * an {@link TimelineDomain} object
+ * @throws IOException
+ * @throws YarnException
+ */
+ @Public
+ public abstract void putDomain(
+ TimelineDomain domain) throws IOException, YarnException;
+
+ /**
+ * <p>
* Get a delegation token so as to be able to talk to the timeline server in a
* secure way.
* </p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index f383a8a..13ce074 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -50,6 +50,8 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
@@ -84,11 +86,15 @@ public class TimelineClientImpl extends TimelineClient {
public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
private static Options opts;
+ private static final String ENTITY_DATA_TYPE = "entity";
+ private static final String DOMAIN_DATA_TYPE = "domain";
static {
opts = new Options();
- opts.addOption("put", true, "Put the TimelineEntities in a JSON file");
+ opts.addOption("put", true, "Put the timeline entities/domain in a JSON file");
opts.getOption("put").setArgName("Path to the JSON file");
+ opts.addOption(ENTITY_DATA_TYPE, false, "Specify the JSON file contains the entities");
+ opts.addOption(DOMAIN_DATA_TYPE, false, "Specify the JSON file contains the domain");
opts.addOption("help", false, "Print usage");
}
@@ -150,9 +156,27 @@ public class TimelineClientImpl extends TimelineClient {
}
TimelineEntities entitiesContainer = new TimelineEntities();
entitiesContainer.addEntities(Arrays.asList(entities));
+ ClientResponse resp = doPosting(entitiesContainer, null);
+ return resp.getEntity(TimelinePutResponse.class);
+ }
+
+
+ @Override
+ public void putDomain(TimelineDomain domain) throws IOException,
+ YarnException {
+ if (!isEnabled) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Nothing will be put because timeline service is not enabled");
+ }
+ return;
+ }
+ doPosting(domain, "domain");
+ }
+
+ private ClientResponse doPosting(Object obj, String path) throws IOException, YarnException {
ClientResponse resp;
try {
- resp = doPostingEntities(entitiesContainer);
+ resp = doPostingObject(obj, path);
} catch (RuntimeException re) {
// runtime exception is expected if the client cannot connect the server
String msg =
@@ -172,7 +196,7 @@ public class TimelineClientImpl extends TimelineClient {
}
throw new YarnException(msg);
}
- return resp.getEntity(TimelinePutResponse.class);
+ return resp;
}
@Override
@@ -184,11 +208,22 @@ public class TimelineClientImpl extends TimelineClient {
@Private
@VisibleForTesting
- public ClientResponse doPostingEntities(TimelineEntities entities) {
+ public ClientResponse doPostingObject(Object object, String path) {
WebResource webResource = client.resource(resURI);
- return webResource.accept(MediaType.APPLICATION_JSON)
- .type(MediaType.APPLICATION_JSON)
- .post(ClientResponse.class, entities);
+ if (path != null) {
+ webResource.path(path);
+ }
+ if (path == null) {
+ return webResource.accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class, object);
+ } else if (path.equals("domain")) {
+ return webResource.path(path).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class, object);
+ } else {
+ throw new YarnRuntimeException("Unknown resource type");
+ }
}
private static class PseudoAuthenticatedURLConnectionFactory
@@ -334,8 +369,13 @@ public class TimelineClientImpl extends TimelineClient {
if (cliParser.hasOption("put")) {
String path = cliParser.getOptionValue("put");
if (path != null && path.length() > 0) {
- putTimelineEntitiesInJSONFile(path);
- return;
+ if (cliParser.hasOption(ENTITY_DATA_TYPE)) {
+ putTimelineDataInJSONFile(path, ENTITY_DATA_TYPE);
+ return;
+ } else if (cliParser.hasOption(DOMAIN_DATA_TYPE)) {
+ putTimelineDataInJSONFile(path, DOMAIN_DATA_TYPE);
+ return;
+ }
}
}
printUsage();
@@ -345,22 +385,28 @@ public class TimelineClientImpl extends TimelineClient {
* Put timeline data in a JSON file via command line.
*
* @param path
- * path to the {@link TimelineEntities} JSON file
+ * path to the timeline data JSON file
+ * @param type
+ * the type of the timeline data in the JSON file
*/
- private static void putTimelineEntitiesInJSONFile(String path) {
+ private static void putTimelineDataInJSONFile(String path, String type) {
File jsonFile = new File(path);
if (!jsonFile.exists()) {
- System.out.println("Error: File [" + jsonFile.getAbsolutePath()
- + "] doesn't exist");
+ LOG.error("File [" + jsonFile.getAbsolutePath() + "] doesn't exist");
return;
}
ObjectMapper mapper = new ObjectMapper();
YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
TimelineEntities entities = null;
+ TimelineDomains domains = null;
try {
- entities = mapper.readValue(jsonFile, TimelineEntities.class);
+ if (type.equals(ENTITY_DATA_TYPE)) {
+ entities = mapper.readValue(jsonFile, TimelineEntities.class);
+ } else if (type.equals(DOMAIN_DATA_TYPE)){
+ domains = mapper.readValue(jsonFile, TimelineDomains.class);
+ }
} catch (Exception e) {
- System.err.println("Error: " + e.getMessage());
+ LOG.error("Error when reading " + e.getMessage());
e.printStackTrace(System.err);
return;
}
@@ -376,21 +422,37 @@ public class TimelineClientImpl extends TimelineClient {
UserGroupInformation.getCurrentUser().getUserName());
UserGroupInformation.getCurrentUser().addToken(token);
}
- TimelinePutResponse response = client.putEntities(
- entities.getEntities().toArray(
- new TimelineEntity[entities.getEntities().size()]));
- if (response.getErrors().size() == 0) {
- System.out.println("Timeline data is successfully put");
- } else {
- for (TimelinePutResponse.TimelinePutError error : response.getErrors()) {
- System.out.println("TimelineEntity [" + error.getEntityType() + ":" +
- error.getEntityId() + "] is not successfully put. Error code: " +
- error.getErrorCode());
+ if (type.equals(ENTITY_DATA_TYPE)) {
+ TimelinePutResponse response = client.putEntities(
+ entities.getEntities().toArray(
+ new TimelineEntity[entities.getEntities().size()]));
+ if (response.getErrors().size() == 0) {
+ LOG.info("Timeline entities are successfully put");
+ } else {
+ for (TimelinePutResponse.TimelinePutError error : response.getErrors()) {
+ LOG.error("TimelineEntity [" + error.getEntityType() + ":" +
+ error.getEntityId() + "] is not successfully put. Error code: " +
+ error.getErrorCode());
+ }
+ }
+ } else if (type.equals(DOMAIN_DATA_TYPE)) {
+ boolean hasError = false;
+ for (TimelineDomain domain : domains.getDomains()) {
+ try {
+ client.putDomain(domain);
+ } catch (Exception e) {
+ LOG.error("Error when putting domain " + domain.getId(), e);
+ hasError = true;
+ }
+ }
+ if (!hasError) {
+ LOG.info("Timeline domains are successfully put");
}
}
+ } catch(RuntimeException e) {
+ LOG.error("Error when putting the timeline data", e);
} catch (Exception e) {
- System.err.println("Error: " + e.getMessage());
- e.printStackTrace(System.err);
+ LOG.error("Error when putting the timeline data", e);
} finally {
client.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
index 7813b5d..2be8160 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
@@ -161,4 +161,42 @@ public class TestTimelineRecords {
Assert.assertEquals(error2.getErrorCode(), e.getErrorCode());
}
+ @Test
+ public void testTimelineDomain() throws Exception {
+ TimelineDomains domains = new TimelineDomains();
+
+ TimelineDomain domain = null;
+ for (int i = 0; i < 2; ++i) {
+ domain = new TimelineDomain();
+ domain.setId("test id " + (i + 1));
+ domain.setDescription("test description " + (i + 1));
+ domain.setOwner("test owner " + (i + 1));
+ domain.setReaders("test_reader_user_" + (i + 1) +
+ " test_reader_group+" + (i + 1));
+ domain.setWriters("test_writer_user_" + (i + 1) +
+ " test_writer_group+" + (i + 1));
+ domain.setCreatedTime(0L);
+ domain.setModifiedTime(1L);
+ domains.addDomain(domain);
+ }
+ LOG.info("Domain in JSON:");
+ LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(domains, true));
+
+ Assert.assertEquals(2, domains.getDomains().size());
+
+ for (int i = 0; i < domains.getDomains().size(); ++i) {
+ domain = domains.getDomains().get(i);
+ Assert.assertEquals("test id " + (i + 1), domain.getId());
+ Assert.assertEquals("test description " + (i + 1),
+ domain.getDescription());
+ Assert.assertEquals("test owner " + (i + 1), domain.getOwner());
+ Assert.assertEquals("test_reader_user_" + (i + 1) +
+ " test_reader_group+" + (i + 1), domain.getReaders());
+ Assert.assertEquals("test_writer_user_" + (i + 1) +
+ " test_writer_group+" + (i + 1), domain.getWriters());
+ Assert.assertEquals(new Long(0L), domain.getCreatedTime());
+ Assert.assertEquals(new Long(1L), domain.getModifiedTime());
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index 3c5272a..9756cde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -27,16 +27,16 @@ import static org.mockito.Mockito.when;
import java.net.ConnectException;
-import org.junit.Assert;
-
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -63,7 +63,7 @@ public class TestTimelineClient {
@Test
public void testPostEntities() throws Exception {
- mockClientResponse(client, ClientResponse.Status.OK, false, false);
+ mockEntityClientResponse(client, ClientResponse.Status.OK, false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
Assert.assertEquals(0, response.getErrors().size());
@@ -74,7 +74,7 @@ public class TestTimelineClient {
@Test
public void testPostEntitiesWithError() throws Exception {
- mockClientResponse(client, ClientResponse.Status.OK, true, false);
+ mockEntityClientResponse(client, ClientResponse.Status.OK, true, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
Assert.assertEquals(1, response.getErrors().size());
@@ -91,7 +91,7 @@ public class TestTimelineClient {
@Test
public void testPostEntitiesNoResponse() throws Exception {
- mockClientResponse(
+ mockEntityClientResponse(
client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
try {
client.putEntities(generateEntity());
@@ -104,7 +104,7 @@ public class TestTimelineClient {
@Test
public void testPostEntitiesConnectionRefused() throws Exception {
- mockClientResponse(client, null, false, true);
+ mockEntityClientResponse(client, null, false, true);
try {
client.putEntities(generateEntity());
Assert.fail("RuntimeException is expected");
@@ -118,7 +118,7 @@ public class TestTimelineClient {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
TimelineClientImpl client = createTimelineClient(conf);
- mockClientResponse(
+ mockEntityClientResponse(
client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
@@ -137,7 +137,7 @@ public class TestTimelineClient {
// Make sure default value is pickup up
conf.unset(YarnConfiguration.TIMELINE_SERVICE_ENABLED);
TimelineClientImpl client = createTimelineClient(conf);
- mockClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR,
+ mockEntityClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR,
false, false);
try {
TimelinePutResponse response = client.putEntities(generateEntity());
@@ -148,16 +148,50 @@ public class TestTimelineClient {
}
}
- private static ClientResponse mockClientResponse(TimelineClientImpl client,
- ClientResponse.Status status, boolean hasError, boolean hasRuntimeError) {
+ @Test
+ public void testPutDomain() throws Exception {
+ mockDomainClientResponse(client, ClientResponse.Status.OK, false);
+ try {
+ client.putDomain(generateDomain());
+ } catch (YarnException e) {
+ Assert.fail("Exception is not expected");
+ }
+ }
+
+ @Test
+ public void testPutDomainNoResponse() throws Exception {
+ mockDomainClientResponse(client, ClientResponse.Status.FORBIDDEN, false);
+ try {
+ client.putDomain(generateDomain());
+ Assert.fail("Exception is expected");
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Failed to get the response from the timeline server."));
+ }
+ }
+
+ @Test
+ public void testPutDomainConnectionRefused() throws Exception {
+ mockDomainClientResponse(client, null, true);
+ try {
+ client.putDomain(generateDomain());
+ Assert.fail("RuntimeException is expected");
+ } catch (RuntimeException re) {
+ Assert.assertTrue(re instanceof ClientHandlerException);
+ }
+ }
+
+ private static ClientResponse mockEntityClientResponse(
+ TimelineClientImpl client, ClientResponse.Status status,
+ boolean hasError, boolean hasRuntimeError) {
ClientResponse response = mock(ClientResponse.class);
if (hasRuntimeError) {
doThrow(new ClientHandlerException(new ConnectException())).when(client)
- .doPostingEntities(any(TimelineEntities.class));
+ .doPostingObject(any(TimelineEntities.class), any(String.class));
return response;
}
doReturn(response).when(client)
- .doPostingEntities(any(TimelineEntities.class));
+ .doPostingObject(any(TimelineEntities.class), any(String.class));
when(response.getClientResponseStatus()).thenReturn(status);
TimelinePutResponse.TimelinePutError error =
new TimelinePutResponse.TimelinePutError();
@@ -172,6 +206,21 @@ public class TestTimelineClient {
return response;
}
+ private static ClientResponse mockDomainClientResponse(
+ TimelineClientImpl client, ClientResponse.Status status,
+ boolean hasRuntimeError) {
+ ClientResponse response = mock(ClientResponse.class);
+ if (hasRuntimeError) {
+ doThrow(new ClientHandlerException(new ConnectException())).when(client)
+ .doPostingObject(any(TimelineDomain.class), any(String.class));
+ return response;
+ }
+ doReturn(response).when(client)
+ .doPostingObject(any(TimelineDomain.class), any(String.class));
+ when(response.getClientResponseStatus()).thenReturn(status);
+ return response;
+ }
+
private static TimelineEntity generateEntity() {
TimelineEntity entity = new TimelineEntity();
entity.setEntityId("entity id");
@@ -194,6 +243,18 @@ public class TestTimelineClient {
return entity;
}
+ public static TimelineDomain generateDomain() {
+ TimelineDomain domain = new TimelineDomain();
+ domain.setId("namesapce id");
+ domain.setDescription("domain description");
+ domain.setOwner("domain owner");
+ domain.setReaders("domain_reader");
+ domain.setWriters("domain_writer");
+ domain.setCreatedTime(0L);
+ domain.setModifiedTime(1L);
+ return domain;
+ }
+
private static TimelineClientImpl createTimelineClient(
YarnConfiguration conf) {
TimelineClientImpl client =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index b0feac1..4cf2708 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -145,6 +147,14 @@ public class LeveldbTimelineStore extends AbstractService
private static final byte[] INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN =
"z".getBytes();
+ private static final byte[] DOMAIN_ENTRY_PREFIX = "d".getBytes();
+ private static final byte[] OWNER_LOOKUP_PREFIX = "o".getBytes();
+ private static final byte[] DESCRIPTION_COLUMN = "d".getBytes();
+ private static final byte[] OWNER_COLUMN = "o".getBytes();
+ private static final byte[] READER_COLUMN = "r".getBytes();
+ private static final byte[] WRITER_COLUMN = "w".getBytes();
+ private static final byte[] TIMESTAMP_COLUMN = "t".getBytes();
+
private static final byte[] EMPTY_BYTES = new byte[0];
private static final String TIMELINE_STORE_VERSION_KEY = "timeline-store-version";
@@ -1558,5 +1568,209 @@ public class LeveldbTimelineStore extends AbstractService
throw new IOException(incompatibleMessage);
}
}
-
+
+ //TODO: make data retention work with the domain data as well
+ @Override
+ public void put(TimelineDomain domain) throws IOException {
+ WriteBatch writeBatch = null;
+ try {
+ writeBatch = db.createWriteBatch();
+ if (domain.getId() == null || domain.getId().length() == 0) {
+ throw new IllegalArgumentException("Domain doesn't have an ID");
+ }
+ if (domain.getOwner() == null || domain.getOwner().length() == 0) {
+ throw new IllegalArgumentException("Domain doesn't have an owner.");
+ }
+
+ // Write description
+ byte[] domainEntryKey = createDomainEntryKey(
+ domain.getId(), DESCRIPTION_COLUMN);
+ byte[] ownerLookupEntryKey = createOwnerLookupKey(
+ domain.getOwner(), domain.getId(), DESCRIPTION_COLUMN);
+ if (domain.getDescription() != null) {
+ writeBatch.put(domainEntryKey, domain.getDescription().getBytes());
+ writeBatch.put(ownerLookupEntryKey, domain.getDescription().getBytes());
+ } else {
+ writeBatch.put(domainEntryKey, EMPTY_BYTES);
+ writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
+ }
+
+ // Write owner
+ domainEntryKey = createDomainEntryKey(domain.getId(), OWNER_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ domain.getOwner(), domain.getId(), OWNER_COLUMN);
+ // Null check for owner is done before
+ writeBatch.put(domainEntryKey, domain.getOwner().getBytes());
+ writeBatch.put(ownerLookupEntryKey, domain.getOwner().getBytes());
+
+ // Write readers
+ domainEntryKey = createDomainEntryKey(domain.getId(), READER_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ domain.getOwner(), domain.getId(), READER_COLUMN);
+ if (domain.getReaders() != null && domain.getReaders().length() > 0) {
+ writeBatch.put(domainEntryKey, domain.getReaders().getBytes());
+ writeBatch.put(ownerLookupEntryKey, domain.getReaders().getBytes());
+ } else {
+ writeBatch.put(domainEntryKey, EMPTY_BYTES);
+ writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
+ }
+
+ // Write writers
+ domainEntryKey = createDomainEntryKey(domain.getId(), WRITER_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ domain.getOwner(), domain.getId(), WRITER_COLUMN);
+ if (domain.getWriters() != null && domain.getWriters().length() > 0) {
+ writeBatch.put(domainEntryKey, domain.getWriters().getBytes());
+ writeBatch.put(ownerLookupEntryKey, domain.getWriters().getBytes());
+ } else {
+ writeBatch.put(domainEntryKey, EMPTY_BYTES);
+ writeBatch.put(ownerLookupEntryKey, EMPTY_BYTES);
+ }
+
+ // Write creation time and modification time
+ // We put both timestamps together because they are always retrieved
+ // together, and store them in the same way as we did for the entity's
+ // start time and insert time.
+ domainEntryKey = createDomainEntryKey(domain.getId(), TIMESTAMP_COLUMN);
+ ownerLookupEntryKey = createOwnerLookupKey(
+ domain.getOwner(), domain.getId(), TIMESTAMP_COLUMN);
+ long currentTimestamp = System.currentTimeMillis();
+ byte[] timestamps = db.get(domainEntryKey);
+ if (timestamps == null) {
+ timestamps = new byte[16];
+ writeReverseOrderedLong(currentTimestamp, timestamps, 0);
+ writeReverseOrderedLong(currentTimestamp, timestamps, 8);
+ } else {
+ writeReverseOrderedLong(currentTimestamp, timestamps, 8);
+ }
+ writeBatch.put(domainEntryKey, timestamps);
+ writeBatch.put(ownerLookupEntryKey, timestamps);
+ db.write(writeBatch);
+ } finally {
+ IOUtils.cleanup(LOG, writeBatch);
+ }
+ }
+
+ /**
+ * Creates a domain entity key with column name suffix,
+ * of the form DOMAIN_ENTRY_PREFIX + domain id + column name.
+ */
+ private static byte[] createDomainEntryKey(String domainId,
+ byte[] columnName) throws IOException {
+ return KeyBuilder.newInstance().add(DOMAIN_ENTRY_PREFIX)
+ .add(domainId).add(columnName).getBytes();
+ }
+
+ /**
+ * Creates an owner lookup key with column name suffix,
+ * of the form OWNER_LOOKUP_PREFIX + owner + domain id + column name.
+ */
+ private static byte[] createOwnerLookupKey(
+ String owner, String domainId, byte[] columnName) throws IOException {
+ return KeyBuilder.newInstance().add(OWNER_LOOKUP_PREFIX)
+ .add(owner).add(domainId).add(columnName).getBytes();
+ }
+
+ @Override
+ public TimelineDomain getDomain(String domainId)
+ throws IOException {
+ DBIterator iterator = null;
+ try {
+ byte[] prefix = KeyBuilder.newInstance()
+ .add(DOMAIN_ENTRY_PREFIX).add(domainId).getBytesForLookup();
+ iterator = db.iterator();
+ iterator.seek(prefix);
+ return getTimelineDomain(iterator, domainId, prefix);
+ } finally {
+ IOUtils.cleanup(LOG, iterator);
+ }
+ }
+
+ @Override
+ public TimelineDomains getDomains(String owner)
+ throws IOException {
+ DBIterator iterator = null;
+ try {
+ byte[] prefix = KeyBuilder.newInstance()
+ .add(OWNER_LOOKUP_PREFIX).add(owner).getBytesForLookup();
+ List<TimelineDomain> domains = new ArrayList<TimelineDomain>();
+ for (iterator = db.iterator(), iterator.seek(prefix);
+ iterator.hasNext();) {
+ byte[] key = iterator.peekNext().getKey();
+ if (!prefixMatches(prefix, prefix.length, key)) {
+ break;
+ }
+ // Iterator to parse the rows of an individual domain
+ KeyParser kp = new KeyParser(key, prefix.length);
+ String domainId = kp.getNextString();
+ byte[] prefixExt = KeyBuilder.newInstance().add(OWNER_LOOKUP_PREFIX)
+ .add(owner).add(domainId).getBytesForLookup();
+ TimelineDomain domainToReturn =
+ getTimelineDomain(iterator, domainId, prefixExt);
+ if (domainToReturn != null) {
+ domains.add(domainToReturn);
+ }
+ }
+ // Sort the domains to return
+ Collections.sort(domains, new Comparator<TimelineDomain>() {
+ @Override
+ public int compare(
+ TimelineDomain domain1, TimelineDomain domain2) {
+ int result = domain2.getCreatedTime().compareTo(
+ domain1.getCreatedTime());
+ if (result == 0) {
+ return domain2.getModifiedTime().compareTo(
+ domain1.getModifiedTime());
+ } else {
+ return result;
+ }
+ }
+ });
+ TimelineDomains domainsToReturn = new TimelineDomains();
+ domainsToReturn.addDomains(domains);
+ return domainsToReturn;
+ } finally {
+ IOUtils.cleanup(LOG, iterator);
+ }
+ }
+
+ private static TimelineDomain getTimelineDomain(
+ DBIterator iterator, String domainId, byte[] prefix) throws IOException {
+ // Iterate over all the rows whose key starts with prefix to retrieve the
+ // domain information.
+ TimelineDomain domain = new TimelineDomain();
+ domain.setId(domainId);
+ boolean noRows = true;
+ for (; iterator.hasNext(); iterator.next()) {
+ byte[] key = iterator.peekNext().getKey();
+ if (!prefixMatches(prefix, prefix.length, key)) {
+ break;
+ }
+ if (noRows) {
+ noRows = false;
+ }
+ byte[] value = iterator.peekNext().getValue();
+ if (value != null && value.length > 0) {
+ if (key[prefix.length] == DESCRIPTION_COLUMN[0]) {
+ domain.setDescription(new String(value));
+ } else if (key[prefix.length] == OWNER_COLUMN[0]) {
+ domain.setOwner(new String(value));
+ } else if (key[prefix.length] == READER_COLUMN[0]) {
+ domain.setReaders(new String(value));
+ } else if (key[prefix.length] == WRITER_COLUMN[0]) {
+ domain.setWriters(new String(value));
+ } else if (key[prefix.length] == TIMESTAMP_COLUMN[0]) {
+ domain.setCreatedTime(readReverseOrderedLong(value, 0));
+ domain.setModifiedTime(readReverseOrderedLong(value, 8));
+ } else {
+ LOG.error("Unrecognized domain column: " + key[prefix.length]);
+ }
+ }
+ }
+ if (noRows) {
+ return null;
+ } else {
+ return domain;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
index 099d5ef..4b6ec63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.yarn.server.timeline;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -40,6 +43,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
@@ -61,6 +66,10 @@ public class MemoryTimelineStore
new HashMap<EntityIdentifier, TimelineEntity>();
private Map<EntityIdentifier, Long> entityInsertTimes =
new HashMap<EntityIdentifier, Long>();
+ private Map<String, TimelineDomain> domainsById =
+ new HashMap<String, TimelineDomain>();
+ private Map<String, Set<TimelineDomain>> domainsByOwner =
+ new HashMap<String, Set<TimelineDomain>>();
public MemoryTimelineStore() {
super(MemoryTimelineStore.class.getName());
@@ -211,6 +220,58 @@ public class MemoryTimelineStore
}
@Override
+ public TimelineDomain getDomain(String domainId)
+ throws IOException {
+ TimelineDomain domain = domainsById.get(domainId);
+ if (domain == null) {
+ return null;
+ } else {
+ return createTimelineDomain(
+ domain.getId(),
+ domain.getDescription(),
+ domain.getOwner(),
+ domain.getReaders(),
+ domain.getWriters(),
+ domain.getCreatedTime(),
+ domain.getModifiedTime());
+ }
+ }
+
+ @Override
+ public TimelineDomains getDomains(String owner)
+ throws IOException {
+ List<TimelineDomain> domains = new ArrayList<TimelineDomain>();
+ for (TimelineDomain domain : domainsByOwner.get(owner)) {
+ TimelineDomain domainToReturn = createTimelineDomain(
+ domain.getId(),
+ domain.getDescription(),
+ domain.getOwner(),
+ domain.getReaders(),
+ domain.getWriters(),
+ domain.getCreatedTime(),
+ domain.getModifiedTime());
+ domains.add(domainToReturn);
+ }
+ Collections.sort(domains, new Comparator<TimelineDomain>() {
+ @Override
+ public int compare(
+ TimelineDomain domain1, TimelineDomain domain2) {
+ int result = domain2.getCreatedTime().compareTo(
+ domain1.getCreatedTime());
+ if (result == 0) {
+ return domain2.getModifiedTime().compareTo(
+ domain1.getModifiedTime());
+ } else {
+ return result;
+ }
+ }
+ });
+ TimelineDomains domainsToReturn = new TimelineDomains();
+ domainsToReturn.addDomains(domains);
+ return domainsToReturn;
+ }
+
+ @Override
public synchronized TimelinePutResponse put(TimelineEntities data) {
TimelinePutResponse response = new TimelinePutResponse();
for (TimelineEntity entity : data.getEntities()) {
@@ -308,6 +369,44 @@ public class MemoryTimelineStore
return response;
}
+ public void put(TimelineDomain domain) throws IOException {
+ TimelineDomain domainToReplace =
+ domainsById.get(domain.getId());
+ long currentTimestamp = System.currentTimeMillis();
+ TimelineDomain domainToStore = createTimelineDomain(
+ domain.getId(), domain.getDescription(), domain.getOwner(),
+ domain.getReaders(), domain.getWriters(),
+ (domainToReplace == null ?
+ currentTimestamp : domainToReplace.getCreatedTime()),
+ currentTimestamp);
+ domainsById.put(domainToStore.getId(), domainToStore);
+ Set<TimelineDomain> domainsByOneOwner =
+ domainsByOwner.get(domainToStore.getOwner());
+ if (domainsByOneOwner == null) {
+ domainsByOneOwner = new HashSet<TimelineDomain>();
+ domainsByOwner.put(domainToStore.getOwner(), domainsByOneOwner);
+ }
+ if (domainToReplace != null) {
+ domainsByOneOwner.remove(domainToReplace);
+ }
+ domainsByOneOwner.add(domainToStore);
+ }
+
+ private static TimelineDomain createTimelineDomain(
+ String id, String description, String owner,
+ String readers, String writers,
+ Long createdTime, Long modifiedTime) {
+ TimelineDomain domainToStore = new TimelineDomain();
+ domainToStore.setId(id);
+ domainToStore.setDescription(description);
+ domainToStore.setOwner(owner);
+ domainToStore.setReaders(readers);
+ domainToStore.setWriters(writers);
+ domainToStore.setCreatedTime(createdTime);
+ domainToStore.setModifiedTime(modifiedTime);
+ return domainToStore;
+ }
+
private static TimelineEntity maskFields(
TimelineEntity entity, EnumSet<Field> fields) {
// Conceal the fields that are not going to be exposed
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
index e68e860..bd83149 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineDataManager.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
@@ -286,6 +288,78 @@ public class TimelineDataManager {
return response;
}
+ /**
+ * Add or update an domain. If the domain already exists, only the owner
+ * and the admin can update it.
+ */
+ public void putDomain(TimelineDomain domain,
+ UserGroupInformation callerUGI) throws YarnException, IOException {
+ TimelineDomain existingDomain =
+ store.getDomain(domain.getId());
+ if (existingDomain != null) {
+ if (!timelineACLsManager.checkAccess(callerUGI, existingDomain)) {
+ throw new YarnException(callerUGI.getShortUserName() +
+ " is not allowed to override an existing domain " +
+ existingDomain.getId());
+ }
+ // Set it again in case ACLs are not enabled: The domain can be
+ // modified by every body, but the owner is not changed.
+ domain.setOwner(existingDomain.getOwner());
+ }
+ store.put(domain);
+ }
+
+ /**
+ * Get a single domain of the particular ID. If callerUGI is not the owner
+ * or the admin of the domain, we need to hide the details from him, and
+ * only allow him to see the ID.
+ */
+ public TimelineDomain getDomain(String domainId,
+ UserGroupInformation callerUGI) throws YarnException, IOException {
+ TimelineDomain domain = store.getDomain(domainId);
+ if (domain != null) {
+ if (timelineACLsManager.checkAccess(callerUGI, domain)) {
+ return domain;
+ } else {
+ hideDomainDetails(domain);
+ return domain;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get all the domains that belong to the given owner. If callerUGI is not
+ * the owner or the admin of the domain, we need to hide the details from
+ * him, and only allow him to see the ID.
+ */
+ public TimelineDomains getDomains(String owner,
+ UserGroupInformation callerUGI) throws YarnException, IOException {
+ TimelineDomains domains = store.getDomains(owner);
+ boolean hasAccess = true;
+ boolean isChecked = false;
+ for (TimelineDomain domain : domains.getDomains()) {
+ // The owner for each domain is the same, just need to check on
+ if (!isChecked) {
+ hasAccess = timelineACLsManager.checkAccess(callerUGI, domain);
+ isChecked = true;
+ }
+ if (!hasAccess) {
+ hideDomainDetails(domain);
+ }
+ }
+ return domains;
+ }
+
+ private static void hideDomainDetails(TimelineDomain domain) {
+ domain.setDescription(null);
+ domain.setOwner(null);
+ domain.setReaders(null);
+ domain.setWriters(null);
+ domain.setCreatedTime(null);
+ domain.setModifiedTime(null);
+ }
+
private static boolean extendFields(EnumSet<Field> fieldEnums) {
boolean modified = false;
if (fieldEnums != null && !fieldEnums.contains(Field.PRIMARY_FILTERS)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java
index 23bca34..aba1ba2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineReader.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
/**
* This interface is for retrieving timeline information.
@@ -152,4 +154,25 @@ public interface TimelineReader {
TimelineEvents getEntityTimelines(String entityType,
SortedSet<String> entityIds, Long limit, Long windowStart,
Long windowEnd, Set<String> eventTypes) throws IOException;
+
+ /**
+ * This method retrieves the domain information for a given ID.
+ *
+ * @return a {@link TimelineDomain} object.
+ * @throws IOException
+ */
+ TimelineDomain getDomain(
+ String domainId) throws IOException;
+
+ /**
+ * This method retrieves all the domains that belong to a given owner.
+ * The domains are sorted according to the created time firstly and the
+ * modified time secondly in descending order.
+ *
+ * @param owner
+ * the domain owner
+ * @return an {@link TimelineDomains} object.
+ * @throws IOException
+ */
+ TimelineDomains getDomains(String owner) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java
index a3e5aeb..7292b5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/TimelineWriter.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.yarn.server.timeline;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
-import java.io.IOException;
-
/**
* This interface is for storing timeline information.
*/
@@ -37,10 +38,21 @@ public interface TimelineWriter {
* individual put request objects will be reported in the response.
*
* @param data
- * An {@link TimelineEntities} object.
- * @return An {@link TimelinePutResponse} object.
+ * a {@link TimelineEntities} object.
+ * @return a {@link TimelinePutResponse} object.
* @throws IOException
*/
TimelinePutResponse put(TimelineEntities data) throws IOException;
+ /**
+ * Store domain information to the timeline store. If A domain of the
+ * same ID already exists in the timeline store, it will be COMPLETELY updated
+ * with the given domain.
+ *
+ * @param domain
+ * a {@link TimelineDomain} object
+ * @throws IOException
+ */
+ void put(TimelineDomain domain) throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
index 10e62d2..6cf7b51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
@@ -81,6 +82,31 @@ public class TimelineACLsManager {
return false;
}
+ public boolean checkAccess(UserGroupInformation callerUGI,
+ TimelineDomain domain) throws YarnException, IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Verifying the access of "
+ + (callerUGI == null ? null : callerUGI.getShortUserName())
+ + " on the timeline domain " + domain);
+ }
+
+ if (!adminAclsManager.areACLsEnabled()) {
+ return true;
+ }
+
+ String owner = domain.getOwner();
+ if (owner == null || owner.length() == 0) {
+ throw new YarnException("Owner information of the timeline domain "
+ + domain.getId() + " is corrupted.");
+ }
+ if (callerUGI != null
+ && (adminAclsManager.isAdmin(callerUGI) ||
+ callerUGI.getShortUserName().equals(owner))) {
+ return true;
+ }
+ return false;
+ }
+
@Private
@VisibleForTesting
public AdminACLsManager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index c5e6d49..f290930 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -32,6 +33,7 @@ import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
@@ -40,6 +42,7 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
@@ -53,7 +56,10 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
@@ -259,6 +265,100 @@ public class TimelineWebServices {
}
}
+ /**
+ * Store the given domain into the timeline store, and return the errors
+ * that happen during storing.
+ */
+ @PUT
+ @Path("/domain")
+ @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public Response putDomain(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ TimelineDomain domain) {
+ init(res);
+ UserGroupInformation callerUGI = getUser(req);
+ if (callerUGI == null) {
+ String msg = "The owner of the posted timeline domain is not set";
+ LOG.error(msg);
+ throw new ForbiddenException(msg);
+ }
+ domain.setOwner(callerUGI.getShortUserName());
+ try {
+ timelineDataManager.putDomain(domain, callerUGI);
+ } catch (YarnException e) {
+ // The user doesn't have the access to override the existing domain.
+ LOG.error(e.getMessage(), e);
+ throw new ForbiddenException(e);
+ } catch (IOException e) {
+ LOG.error("Error putting domain", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return Response.status(Status.OK).build();
+ }
+
+ /**
+ * Return a single domain of the given domain Id.
+ */
+ @GET
+ @Path("/domain/{domainId}")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelineDomain getDomain(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("domainId") String domainId) {
+ init(res);
+ domainId = parseStr(domainId);
+ if (domainId == null || domainId.length() == 0) {
+ throw new BadRequestException("Domain ID is not specified.");
+ }
+ TimelineDomain domain = null;
+ try {
+ domain = timelineDataManager.getDomain(
+ parseStr(domainId), getUser(req));
+ } catch (Exception e) {
+ LOG.error("Error getting domain", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ if (domain == null) {
+ throw new NotFoundException("Timeline domain ["
+ + domainId + "] is not found");
+ }
+ return domain;
+ }
+
+ /**
+ * Return a list of domains of the given owner.
+ */
+ @GET
+ @Path("/domain")
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelineDomains getDomains(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("owner") String owner) {
+ init(res);
+ owner = parseStr(owner);
+ UserGroupInformation callerUGI = getUser(req);
+ if (owner == null || owner.length() == 0) {
+ if (callerUGI == null) {
+ throw new BadRequestException("Domain owner is not specified.");
+ } else {
+ // By default it's going to list the caller's domains
+ owner = callerUGI.getShortUserName();
+ }
+ }
+ try {
+ return timelineDataManager.getDomains(owner, callerUGI);
+ } catch (Exception e) {
+ LOG.error("Error getting domains", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
private void init(HttpServletResponse response) {
response.setContentType(null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
index 0c6e082..b35a100 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
@@ -69,8 +69,9 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
store = new LeveldbTimelineStore();
store.init(config);
store.start();
- loadTestData();
- loadVerificationData();
+ loadTestEntityData();
+ loadVerificationEntityData();
+ loadTestDomainData();
}
@After
@@ -93,7 +94,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
super.testGetSingleEntity();
((LeveldbTimelineStore)store).clearStartTimeCache();
super.testGetSingleEntity();
- loadTestData();
+ loadTestEntityData();
}
@Test
@@ -257,7 +258,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
assertEquals(0, getEntities("type_2").size());
assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
l).size());
- loadTestData();
+ loadTestEntityData();
assertEquals(0, getEntitiesFromTs("type_1", l).size());
assertEquals(0, getEntitiesFromTs("type_2", l).size());
assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
@@ -309,4 +310,14 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
store.start();
}
+ @Test
+ public void testGetDomain() throws IOException {
+ super.testGetDomain();
+ }
+
+ @Test
+ public void testGetDomains() throws IOException {
+ super.testGetDomains();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
index 1953442..6ba1bfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
@@ -34,8 +34,9 @@ public class TestMemoryTimelineStore extends TimelineStoreTestUtils {
store = new MemoryTimelineStore();
store.init(new YarnConfiguration());
store.start();
- loadTestData();
- loadVerificationData();
+ loadTestEntityData();
+ loadVerificationEntityData();
+ loadTestDomainData();
}
@After
@@ -82,4 +83,14 @@ public class TestMemoryTimelineStore extends TimelineStoreTestUtils {
super.testGetEvents();
}
+ @Test
+ public void testGetDomain() throws IOException {
+ super.testGetDomain();
+ }
+
+ @Test
+ public void testGetDomains() throws IOException {
+ super.testGetDomains();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
index e8a6d83..d31ad73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
@@ -38,11 +38,11 @@ import java.util.TreeSet;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
-import org.apache.hadoop.yarn.server.timeline.NameValuePair;
-import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
public class TimelineStoreTestUtils {
@@ -88,9 +88,9 @@ public class TimelineStoreTestUtils {
protected long beforeTs;
/**
- * Load test data into the given store
+ * Load test entity data into the given store
*/
- protected void loadTestData() throws IOException {
+ protected void loadTestEntityData() throws IOException {
beforeTs = System.currentTimeMillis()-1;
TimelineEntities entities = new TimelineEntities();
Map<String, Set<Object>> primaryFilters =
@@ -184,9 +184,9 @@ public class TimelineStoreTestUtils {
}
/**
- * Load verification data
+ * Load verification entity data
*/
- protected void loadVerificationData() throws Exception {
+ protected void loadVerificationEntityData() throws Exception {
userFilter = new NameValuePair("user", "username");
numericFilter1 = new NameValuePair("appname", Integer.MAX_VALUE);
numericFilter2 = new NameValuePair("long", (long)Integer.MAX_VALUE + 1l);
@@ -263,6 +263,51 @@ public class TimelineStoreTestUtils {
events2.add(ev4);
}
+ private TimelineDomain domain1;
+ private TimelineDomain domain2;
+ private TimelineDomain domain3;
+ private long elapsedTime;
+
+ protected void loadTestDomainData() throws IOException {
+ domain1 = new TimelineDomain();
+ domain1.setId("domain_id_1");
+ domain1.setDescription("description_1");
+ domain1.setOwner("owner_1");
+ domain1.setReaders("reader_user_1 reader_group_1");
+ domain1.setWriters("writer_user_1 writer_group_1");
+ store.put(domain1);
+
+ domain2 = new TimelineDomain();
+ domain2.setId("domain_id_2");
+ domain2.setDescription("description_2");
+ domain2.setOwner("owner_2");
+ domain2.setReaders("reader_user_2 reader_group_2");
+ domain2.setWriters("writer_user_2writer_group_2");
+ store.put(domain2);
+
+ // Wait a second before updating the domain information
+ elapsedTime = 1000;
+ try {
+ Thread.sleep(elapsedTime);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+
+ domain2.setDescription("description_3");
+ domain2.setOwner("owner_3");
+ domain2.setReaders("reader_user_3 reader_group_3");
+ domain2.setWriters("writer_user_3 writer_group_3");
+ store.put(domain2);
+
+ domain3 = new TimelineDomain();
+ domain3.setId("domain_id_4");
+ domain3.setDescription("description_4");
+ domain3.setOwner("owner_1");
+ domain3.setReaders("reader_user_4 reader_group_4");
+ domain3.setWriters("writer_user_4 writer_group_4");
+ store.put(domain3);
+ }
+
public void testGetSingleEntity() throws IOException {
// test getting entity info
verifyEntityInfo(null, null, null, null, null, null,
@@ -519,7 +564,7 @@ public class TimelineStoreTestUtils {
assertEquals(2, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
// check insert time is not overwritten
long beforeTs = this.beforeTs;
- loadTestData();
+ loadTestEntityData();
assertEquals(0, getEntitiesFromTs("type_1", beforeTs).size());
assertEquals(0, getEntitiesFromTs("type_2", beforeTs).size());
assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
@@ -788,4 +833,39 @@ public class TimelineStoreTestUtils {
return event;
}
+ public void testGetDomain() throws IOException {
+ TimelineDomain actualDomain1 =
+ store.getDomain(domain1.getId());
+ verifyDomainInfo(domain1, actualDomain1);
+ assertTrue(actualDomain1.getCreatedTime() > 0);
+ assertTrue(actualDomain1.getModifiedTime() > 0);
+ assertEquals(
+ actualDomain1.getCreatedTime(), actualDomain1.getModifiedTime());
+
+ TimelineDomain actualDomain2 =
+ store.getDomain(domain2.getId());
+ verifyDomainInfo(domain2, actualDomain2);
+ assertEquals("domain_id_2", actualDomain2.getId());
+ assertTrue(actualDomain2.getCreatedTime() > 0);
+ assertTrue(actualDomain2.getModifiedTime() > 0);
+ assertTrue(
+ actualDomain2.getCreatedTime() < actualDomain2.getModifiedTime());
+ }
+
+ public void testGetDomains() throws IOException {
+ TimelineDomains actualDomains =
+ store.getDomains("owner_1");
+ assertEquals(2, actualDomains.getDomains().size());
+ verifyDomainInfo(domain3, actualDomains.getDomains().get(0));
+ verifyDomainInfo(domain1, actualDomains.getDomains().get(1));
+ }
+
+ private static void verifyDomainInfo(
+ TimelineDomain expected, TimelineDomain actual) {
+ assertEquals(expected.getId(), actual.getId());
+ assertEquals(expected.getDescription(), actual.getDescription());
+ assertEquals(expected.getOwner(), actual.getOwner());
+ assertEquals(expected.getReaders(), actual.getReaders());
+ assertEquals(expected.getWriters(), actual.getWriters());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d78b452a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
index 5825e7e..924aa9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
@@ -21,17 +21,17 @@ package org.apache.hadoop.yarn.server.timeline.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.junit.Assert;
import org.junit.Test;
public class TestTimelineACLsManager {
@Test
- public void testYarnACLsNotEnabled() throws Exception {
+ public void testYarnACLsNotEnabledForEntity() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
TimelineACLsManager timelineACLsManager =
@@ -47,7 +47,7 @@ public class TestTimelineACLsManager {
}
@Test
- public void testYarnACLsEnabled() throws Exception {
+ public void testYarnACLsEnabledForEntity() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
@@ -72,7 +72,7 @@ public class TestTimelineACLsManager {
}
@Test
- public void testCorruptedOwnerInfo() throws Exception {
+ public void testCorruptedOwnerInfoForEntity() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
@@ -89,4 +89,59 @@ public class TestTimelineACLsManager {
}
}
+ @Test
+ public void testYarnACLsNotEnabledForDomain() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
+ TimelineACLsManager timelineACLsManager =
+ new TimelineACLsManager(conf);
+ TimelineDomain domain = new TimelineDomain();
+ domain.setOwner("owner");
+ Assert.assertTrue(
+ "Always true when ACLs are not enabled",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("user"), domain));
+ }
+
+ @Test
+ public void testYarnACLsEnabledForDomain() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
+ TimelineACLsManager timelineACLsManager =
+ new TimelineACLsManager(conf);
+ TimelineDomain domain = new TimelineDomain();
+ domain.setOwner("owner");
+ Assert.assertTrue(
+ "Owner should be allowed to access",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("owner"), domain));
+ Assert.assertFalse(
+ "Other shouldn't be allowed to access",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("other"), domain));
+ Assert.assertTrue(
+ "Admin should be allowed to access",
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("admin"), domain));
+ }
+
+ @Test
+ public void testCorruptedOwnerInfoForDomain() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
+ TimelineACLsManager timelineACLsManager =
+ new TimelineACLsManager(conf);
+ TimelineDomain domain = new TimelineDomain();
+ try {
+ timelineACLsManager.checkAccess(
+ UserGroupInformation.createRemoteUser("owner"), domain);
+ Assert.fail("Exception is expected");
+ } catch (YarnException e) {
+ Assert.assertTrue("It's not the exact expected exception", e.getMessage()
+ .contains("is corrupted."));
+ }
+ }
+
}