You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2019/03/12 17:06:48 UTC
[hadoop] branch trunk updated: YARN-9255. Improve recommend
applications order and fix findbugs warnings. Contributed by Eric Yang
This is an automated email from the ASF dual-hosted git repository.
billie pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new aab7b77 YARN-9255. Improve recommend applications order and fix findbugs warnings. Contributed by Eric Yang
aab7b77 is described below
commit aab7b77536ef3feaead85106de85d628fcfebcb1
Author: Billie Rinaldi <bi...@apache.org>
AuthorDate: Tue Mar 12 10:06:09 2019 -0700
YARN-9255. Improve recommend applications order and fix findbugs warnings. Contributed by Eric Yang
---
.../application/AppCatalogSolrClient.java | 65 ++++++++++++++++++----
.../hadoop/yarn/appcatalog/model/AppDetails.java | 12 ++--
.../hadoop/yarn/appcatalog/model/Application.java | 22 ++++++++
.../application/TestAppCatalogSolrClient.java | 22 +++++++-
4 files changed, 103 insertions(+), 18 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java
index 1397722..f3532ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
@@ -64,12 +65,16 @@ public class AppCatalogSolrClient {
Properties properties = new Properties();
try {
properties.load(input);
- urlString = properties.getProperty("solr_url");
+ setSolrUrl(properties.getProperty("solr_url"));
} catch (IOException e) {
LOG.error("Error reading appcatalog configuration: ", e);
}
}
+ private synchronized static void setSolrUrl(String url) {
+ urlString = url;
+ }
+
public SolrClient getSolrClient() {
return new HttpSolrClient.Builder(urlString).build();
}
@@ -79,6 +84,7 @@ public class AppCatalogSolrClient {
SolrClient solr = getSolrClient();
SolrQuery query = new SolrQuery();
query.setQuery("*:*");
+ query.setSort("download_i", ORDER.desc);
query.setFilterQueries("type_s:AppStoreEntry");
query.setRows(40);
QueryResponse response;
@@ -95,8 +101,8 @@ public class AppCatalogSolrClient {
if (d.get("icon_s")!=null) {
entry.setIcon(d.get("icon_s").toString());
}
- entry.setLike(Integer.valueOf(d.get("like_i").toString()));
- entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+ entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+ entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
apps.add(entry);
}
} catch (SolrServerException | IOException e) {
@@ -128,8 +134,8 @@ public class AppCatalogSolrClient {
entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString());
- entry.setLike(Integer.valueOf(d.get("like_i").toString()));
- entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+ entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+ entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
apps.add(entry);
}
} catch (SolrServerException | IOException e) {
@@ -189,8 +195,8 @@ public class AppCatalogSolrClient {
entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString());
- entry.setLike(Integer.valueOf(d.get("like_i").toString()));
- entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+ entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+ entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
Service yarnApp = mapper.readValue(d.get("yarnfile_s").toString(),
Service.class);
String name;
@@ -263,8 +269,8 @@ public class AppCatalogSolrClient {
entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString());
- entry.setLike(Integer.valueOf(d.get("like_i").toString()));
- entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
+ entry.setLike(Integer.parseInt(d.get("like_i").toString()));
+ entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
download = entry.getDownload() + 1;
// Update download count
@@ -303,7 +309,8 @@ public class AppCatalogSolrClient {
s.addField(name, doc.getFieldValues(name));
}
}
- s.setField("download_i", download++);
+ download++;
+ s.setField("download_i", download);
return s;
}
@@ -356,4 +363,42 @@ public class AppCatalogSolrClient {
}
}
+ protected void register(AppStoreEntry app) throws IOException {
+ Collection<SolrInputDocument> docs = new HashSet<SolrInputDocument>();
+ SolrClient solr = getSolrClient();
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ try {
+ SolrInputDocument buffer = new SolrInputDocument();
+ buffer.setField("id", java.util.UUID.randomUUID().toString()
+ .substring(0, 11));
+ buffer.setField("org_s", app.getOrg());
+ buffer.setField("name_s", app.getName());
+ buffer.setField("desc_s", app.getDesc());
+ if (app.getIcon() != null) {
+ buffer.setField("icon_s", app.getIcon());
+ }
+ buffer.setField("type_s", "AppStoreEntry");
+ buffer.setField("like_i", app.getLike());
+ buffer.setField("download_i", app.getDownload());
+
+ // Keep only YARN data model for yarnfile field
+ String yarnFile = mapper.writeValueAsString(app);
+ LOG.info("app:"+yarnFile);
+ Service yarnApp = mapper.readValue(yarnFile, Service.class);
+ buffer.setField("yarnfile_s", mapper.writeValueAsString(yarnApp));
+
+ docs.add(buffer);
+ // Commit Solr changes.
+ UpdateResponse detailsResponse = solr.add(docs);
+ if (detailsResponse.getStatus() != 0) {
+ throw new IOException("Unable to register application " +
+ "in Application Store.");
+ }
+ solr.commit();
+ } catch (SolrServerException | IOException e) {
+ throw new IOException("Unable to register application " +
+ "in Application Store. "+ e.getMessage());
+ }
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java
index 42012cf..d7168fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java
@@ -51,26 +51,26 @@ public class AppDetails {
}
public String[] getPorts() {
- return ports;
+ return ports.clone();
}
public void setPorts(String[] ports2) {
- this.ports = ports2;
+ this.ports = ports2.clone();
}
public String[] getVolumes() {
- return volumes;
+ return volumes.clone();
}
public void setVolumes(String[] volumes) {
- this.volumes = volumes;
+ this.volumes = volumes.clone();
}
public String[] getEnv() {
- return env;
+ return env.clone();
}
public void setEnv(String[] env) {
- this.env = env;
+ this.env = env.clone();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java
index dbc31b1..c99104a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.appcatalog.model;
+import java.util.Objects;
+
import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -64,4 +66,24 @@ public class Application extends Service {
this.icon = icon;
}
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj == this) {
+ return true;
+ } else if ((obj instanceof Application)) {
+ if (((Application) obj).getName().equals(this.getName())
+ && ((Application) obj).getVersion().equals(this.getVersion())
+ && ((Application) obj).getOrganization()
+ .equals(this.getOrganization())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects
+ .hash(this.getName() + this.getVersion() + this.getOrganization());
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java
index a43e0d4..16bf7fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java
@@ -104,9 +104,27 @@ public class TestAppCatalogSolrClient {
@Test
public void testGetRecommendedApps() throws Exception {
- List<AppStoreEntry> expected = spy.getRecommendedApps();
+ AppStoreEntry example = new AppStoreEntry();
+ example.setOrg("jenkins-ci.org");
+ example.setName("jenkins");
+ example.setDesc("World leading open source automation system.");
+ example.setIcon("/css/img/feather.png");
+ example.setDownload(100);
+ spy.register(example);
+ AppStoreEntry example2 = new AppStoreEntry();
+ example2.setOrg("Apache");
+ example2.setName("httpd");
+ example2.setDesc("Apache webserver");
+ example2.setIcon("/css/img/feather.png");
+ example2.setDownload(1);
+ spy.register(example2);
List<AppStoreEntry> actual = spy.getRecommendedApps();
- assertEquals(expected, actual);
+ long previous = 1000L;
+ for (AppStoreEntry app: actual) {
+ assertTrue("Recommend app is not sort by download count.",
+ previous > app.getDownload());
+ previous = app.getDownload();
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org