You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by ey...@apache.org on 2014/07/31 06:05:02 UTC
svn commit: r1614808 [7/7] - in /chukwa/trunk: ./ conf/ contrib/solr/
contrib/solr/logs/ contrib/solr/logs/conf/ contrib/solr/logs/conf/clustering/
contrib/solr/logs/conf/clustering/carrot2/ contrib/solr/logs/conf/lang/
contrib/solr/logs/conf/velocity/...
Added: chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl
URL: http://svn.apache.org/viewvc/chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl (added)
+++ chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl Thu Jul 31 04:04:59 2014
@@ -0,0 +1,337 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+<!--
+ Display the luke request handler with graphs
+ -->
+<xsl:stylesheet
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns="http://www.w3.org/1999/xhtml"
+ version="1.0"
+ >
+ <xsl:output
+ method="html"
+ encoding="UTF-8"
+ media-type="text/html"
+ doctype-public="-//W3C//DTD XHTML 1.0 Strict//EN"
+ doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"
+ />
+
+ <xsl:variable name="title">Solr Luke Request Handler Response</xsl:variable>
+
+ <xsl:template match="/">
+ <html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <link rel="stylesheet" type="text/css" href="solr-admin.css"/>
+ <link rel="icon" href="favicon.ico" type="image/ico"/>
+ <link rel="shortcut icon" href="favicon.ico" type="image/ico"/>
+ <title>
+ <xsl:value-of select="$title"/>
+ </title>
+ <xsl:call-template name="css"/>
+
+ </head>
+ <body>
+ <h1>
+ <xsl:value-of select="$title"/>
+ </h1>
+ <div class="doc">
+ <ul>
+ <xsl:if test="response/lst[@name='index']">
+ <li>
+ <a href="#index">Index Statistics</a>
+ </li>
+ </xsl:if>
+ <xsl:if test="response/lst[@name='fields']">
+ <li>
+ <a href="#fields">Field Statistics</a>
+ <ul>
+ <xsl:for-each select="response/lst[@name='fields']/lst">
+ <li>
+ <a href="#{@name}">
+ <xsl:value-of select="@name"/>
+ </a>
+ </li>
+ </xsl:for-each>
+ </ul>
+ </li>
+ </xsl:if>
+ <xsl:if test="response/lst[@name='doc']">
+ <li>
+ <a href="#doc">Document statistics</a>
+ </li>
+ </xsl:if>
+ </ul>
+ </div>
+ <xsl:if test="response/lst[@name='index']">
+ <h2><a name="index"/>Index Statistics</h2>
+ <xsl:apply-templates select="response/lst[@name='index']"/>
+ </xsl:if>
+ <xsl:if test="response/lst[@name='fields']">
+ <h2><a name="fields"/>Field Statistics</h2>
+ <xsl:apply-templates select="response/lst[@name='fields']"/>
+ </xsl:if>
+ <xsl:if test="response/lst[@name='doc']">
+ <h2><a name="doc"/>Document statistics</h2>
+ <xsl:apply-templates select="response/lst[@name='doc']"/>
+ </xsl:if>
+ </body>
+ </html>
+ </xsl:template>
+
+ <xsl:template match="lst">
+ <xsl:if test="parent::lst">
+ <tr>
+ <td colspan="2">
+ <div class="doc">
+ <xsl:call-template name="list"/>
+ </div>
+ </td>
+ </tr>
+ </xsl:if>
+ <xsl:if test="not(parent::lst)">
+ <div class="doc">
+ <xsl:call-template name="list"/>
+ </div>
+ </xsl:if>
+ </xsl:template>
+
+ <xsl:template name="list">
+ <xsl:if test="count(child::*)>0">
+ <table>
+ <thead>
+ <tr>
+ <th colspan="2">
+ <p>
+ <a name="{@name}"/>
+ </p>
+ <xsl:value-of select="@name"/>
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ <xsl:choose>
+ <xsl:when
+ test="@name='histogram'">
+ <tr>
+ <td colspan="2">
+ <xsl:call-template name="histogram"/>
+ </td>
+ </tr>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:apply-templates/>
+ </xsl:otherwise>
+ </xsl:choose>
+ </tbody>
+ </table>
+ </xsl:if>
+ </xsl:template>
+
+ <xsl:template name="histogram">
+ <div class="doc">
+ <xsl:call-template name="barchart">
+ <xsl:with-param name="max_bar_width">50</xsl:with-param>
+ <xsl:with-param name="iwidth">800</xsl:with-param>
+ <xsl:with-param name="iheight">160</xsl:with-param>
+ <xsl:with-param name="fill">blue</xsl:with-param>
+ </xsl:call-template>
+ </div>
+ </xsl:template>
+
+ <xsl:template name="barchart">
+ <xsl:param name="max_bar_width"/>
+ <xsl:param name="iwidth"/>
+ <xsl:param name="iheight"/>
+ <xsl:param name="fill"/>
+ <xsl:variable name="max">
+ <xsl:for-each select="int">
+ <xsl:sort data-type="number" order="descending"/>
+ <xsl:if test="position()=1">
+ <xsl:value-of select="."/>
+ </xsl:if>
+ </xsl:for-each>
+ </xsl:variable>
+ <xsl:variable name="bars">
+ <xsl:value-of select="count(int)"/>
+ </xsl:variable>
+ <xsl:variable name="bar_width">
+ <xsl:choose>
+ <xsl:when test="$max_bar_width < ($iwidth div $bars)">
+ <xsl:value-of select="$max_bar_width"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:value-of select="$iwidth div $bars"/>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:variable>
+ <table class="histogram">
+ <tbody>
+ <tr>
+ <xsl:for-each select="int">
+ <td>
+ <xsl:value-of select="."/>
+ <div class="histogram">
+ <xsl:attribute name="style">background-color: <xsl:value-of select="$fill"/>; width: <xsl:value-of select="$bar_width"/>px; height: <xsl:value-of select="($iheight*number(.)) div $max"/>px;</xsl:attribute>
+ </div>
+ </td>
+ </xsl:for-each>
+ </tr>
+ <tr>
+ <xsl:for-each select="int">
+ <td>
+ <xsl:value-of select="@name"/>
+ </td>
+ </xsl:for-each>
+ </tr>
+ </tbody>
+ </table>
+ </xsl:template>
+
+ <xsl:template name="keyvalue">
+ <xsl:choose>
+ <xsl:when test="@name">
+ <tr>
+ <td class="name">
+ <xsl:value-of select="@name"/>
+ </td>
+ <td class="value">
+ <xsl:value-of select="."/>
+ </td>
+ </tr>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:value-of select="."/>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:template>
+
+ <xsl:template match="int|bool|long|float|double|uuid|date">
+ <xsl:call-template name="keyvalue"/>
+ </xsl:template>
+
+ <xsl:template match="arr">
+ <tr>
+ <td class="name">
+ <xsl:value-of select="@name"/>
+ </td>
+ <td class="value">
+ <ul>
+ <xsl:for-each select="child::*">
+ <li>
+ <xsl:apply-templates/>
+ </li>
+ </xsl:for-each>
+ </ul>
+ </td>
+ </tr>
+ </xsl:template>
+
+ <xsl:template match="str">
+ <xsl:choose>
+ <xsl:when test="@name='schema' or @name='index' or @name='flags'">
+ <xsl:call-template name="schema"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:call-template name="keyvalue"/>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:template>
+
+ <xsl:template name="schema">
+ <tr>
+ <td class="name">
+ <xsl:value-of select="@name"/>
+ </td>
+ <td class="value">
+ <xsl:if test="contains(.,'unstored')">
+ <xsl:value-of select="."/>
+ </xsl:if>
+ <xsl:if test="not(contains(.,'unstored'))">
+ <xsl:call-template name="infochar2string">
+ <xsl:with-param name="charList">
+ <xsl:value-of select="."/>
+ </xsl:with-param>
+ </xsl:call-template>
+ </xsl:if>
+ </td>
+ </tr>
+ </xsl:template>
+
+ <xsl:template name="infochar2string">
+ <xsl:param name="i">1</xsl:param>
+ <xsl:param name="charList"/>
+
+ <xsl:variable name="char">
+ <xsl:value-of select="substring($charList,$i,1)"/>
+ </xsl:variable>
+ <xsl:choose>
+ <xsl:when test="$char='I'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='I']"/> - </xsl:when>
+ <xsl:when test="$char='T'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='T']"/> - </xsl:when>
+ <xsl:when test="$char='S'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='S']"/> - </xsl:when>
+ <xsl:when test="$char='M'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='M']"/> - </xsl:when>
+ <xsl:when test="$char='V'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='V']"/> - </xsl:when>
+ <xsl:when test="$char='o'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='o']"/> - </xsl:when>
+ <xsl:when test="$char='p'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='p']"/> - </xsl:when>
+ <xsl:when test="$char='O'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='O']"/> - </xsl:when>
+ <xsl:when test="$char='L'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='L']"/> - </xsl:when>
+ <xsl:when test="$char='B'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='B']"/> - </xsl:when>
+ <xsl:when test="$char='C'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='C']"/> - </xsl:when>
+ <xsl:when test="$char='f'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='f']"/> - </xsl:when>
+ <xsl:when test="$char='l'">
+ <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='l']"/> -
+ </xsl:when>
+ </xsl:choose>
+
+ <xsl:if test="not($i>=string-length($charList))">
+ <xsl:call-template name="infochar2string">
+ <xsl:with-param name="i">
+ <xsl:value-of select="$i+1"/>
+ </xsl:with-param>
+ <xsl:with-param name="charList">
+ <xsl:value-of select="$charList"/>
+ </xsl:with-param>
+ </xsl:call-template>
+ </xsl:if>
+ </xsl:template>
+ <xsl:template name="css">
+ <style type="text/css">
+ <![CDATA[
+ td.name {font-style: italic; font-size:80%; }
+ .doc { margin: 0.5em; border: solid grey 1px; }
+ .exp { display: none; font-family: monospace; white-space: pre; }
+ div.histogram { background: none repeat scroll 0%; -moz-background-clip: -moz-initial; -moz-background-origin: -moz-initial; -moz-background-inline-policy: -moz-initial;}
+ table.histogram { width: auto; vertical-align: bottom; }
+ table.histogram td, table.histogram th { text-align: center; vertical-align: bottom; border-bottom: 1px solid #ff9933; width: auto; }
+ ]]>
+ </style>
+ </xsl:template>
+</xsl:stylesheet>
Added: chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl
URL: http://svn.apache.org/viewvc/chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl (added)
+++ chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl Thu Jul 31 04:04:59 2014
@@ -0,0 +1,70 @@
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+<!--
+ Simple transform of Solr query response into Solr Update XML compliant XML.
+ When used in the xslt response writer you will get UpdaateXML as output.
+ But you can also store a query response XML to disk and feed this XML to
+ the XSLTUpdateRequestHandler to index the content. Provided as example only.
+ See http://wiki.apache.org/solr/XsltUpdateRequestHandler for more info
+ -->
+<xsl:stylesheet version='1.0' xmlns:xsl='http://www.w3.org/1999/XSL/Transform'>
+ <xsl:output media-type="text/xml" method="xml" indent="yes"/>
+
+ <xsl:template match='/'>
+ <add>
+ <xsl:apply-templates select="response/result/doc"/>
+ </add>
+ </xsl:template>
+
+ <!-- Ignore score (makes no sense to index) -->
+ <xsl:template match="doc/*[@name='score']" priority="100">
+ </xsl:template>
+
+ <xsl:template match="doc">
+ <xsl:variable name="pos" select="position()"/>
+ <doc>
+ <xsl:apply-templates>
+ <xsl:with-param name="pos"><xsl:value-of select="$pos"/></xsl:with-param>
+ </xsl:apply-templates>
+ </doc>
+ </xsl:template>
+
+ <!-- Flatten arrays to duplicate field lines -->
+ <xsl:template match="doc/arr" priority="100">
+ <xsl:variable name="fn" select="@name"/>
+
+ <xsl:for-each select="*">
+ <xsl:element name="field">
+ <xsl:attribute name="name"><xsl:value-of select="$fn"/></xsl:attribute>
+ <xsl:value-of select="."/>
+ </xsl:element>
+ </xsl:for-each>
+ </xsl:template>
+
+
+ <xsl:template match="doc/*">
+ <xsl:variable name="fn" select="@name"/>
+
+ <xsl:element name="field">
+ <xsl:attribute name="name"><xsl:value-of select="$fn"/></xsl:attribute>
+ <xsl:value-of select="."/>
+ </xsl:element>
+ </xsl:template>
+
+ <xsl:template match="*"/>
+</xsl:stylesheet>
Added: chukwa/trunk/contrib/solr/logs/core.properties
URL: http://svn.apache.org/viewvc/chukwa/trunk/contrib/solr/logs/core.properties?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/contrib/solr/logs/core.properties (added)
+++ chukwa/trunk/contrib/solr/logs/core.properties Thu Jul 31 04:04:59 2014
@@ -0,0 +1 @@
+name=logs
Modified: chukwa/trunk/pom.xml
URL: http://svn.apache.org/viewvc/chukwa/trunk/pom.xml?rev=1614808&r1=1614807&r2=1614808&view=diff
==============================================================================
--- chukwa/trunk/pom.xml (original)
+++ chukwa/trunk/pom.xml Thu Jul 31 04:04:59 2014
@@ -300,6 +300,33 @@
<version>1.2.1</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>4.9.0</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-test-framework</artifactId>
+ <version>4.9.0</version>
+ <type>jar</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<developers>
@@ -1239,7 +1266,7 @@
<artifactId>maven-jxr-plugin</artifactId>
<version>2.3</version>
</plugin>
-<!-- <plugin>
+ <plugin>
<artifactId>maven-pmd-plugin</artifactId>
<version>2.6</version>
<reportSets>
@@ -1262,7 +1289,7 @@
<threshold>Normal</threshold>
<effort>Max</effort>
</configuration>
- </plugin>-->
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
Added: chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java
URL: http://svn.apache.org/viewvc/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java (added)
+++ chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java Thu Jul 31 04:04:59 2014
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.chukwa.datacollection.writer.solr;
+
+import java.util.List;
+
+import org.apache.hadoop.chukwa.Chunk;
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter;
+import org.apache.hadoop.chukwa.datacollection.writer.PipelineableWriter;
+import org.apache.hadoop.chukwa.datacollection.writer.WriterException;
+import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.common.SolrInputDocument;
+
+public class SolrWriter extends PipelineableWriter {
+ private static Logger log = Logger.getLogger(SolrWriter.class);
+ private static CloudSolrServer server;
+ private static String ID = "id";
+ private static String SEQ_ID = "seqId";
+ private static String DATA_TYPE = "type";
+ private static String STREAM_NAME = "stream";
+ private static String TAGS = "tags";
+ private static String SOURCE = "source";
+ private static String DATA = "data";
+
+ public SolrWriter() throws WriterException {
+ init(ChukwaAgent.getStaticConfiguration());
+ }
+
+ @Override
+ public void init(Configuration c) throws WriterException {
+ String serverName = c.get("solr.cloud.address");
+ if (serverName == null) {
+ throw new WriterException("Solr server address is not defined.");
+ }
+ String collection = c.get("solr.collection", "logs");
+ server = new CloudSolrServer(serverName);
+ server.setDefaultCollection(collection);
+ }
+
+ @Override
+ public void close() throws WriterException {
+ }
+
+ @Override
+ public CommitStatus add(List<Chunk> chunks) throws WriterException {
+ CommitStatus rv = ChukwaWriter.COMMIT_OK;
+ for(Chunk chunk : chunks) {
+ try {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField(ID, chunk.getSource() + "_" + chunk.getSeqID());
+ doc.addField(TAGS, chunk.getTags());
+ doc.addField(STREAM_NAME, chunk.getStreamName());
+ doc.addField(SOURCE, chunk.getSource());
+ doc.addField(SEQ_ID, chunk.getSeqID());
+ doc.addField(DATA_TYPE, chunk.getDataType());
+ doc.addField(DATA, new String(chunk.getData()));
+ server.add(doc);
+ server.commit();
+ } catch (Exception e) {
+ log.error(ExceptionUtil.getStackTrace(e));
+ throw new WriterException("Failed to store data to Solr Cloud.");
+ }
+ }
+ if (next != null) {
+ rv = next.add(chunks); //pass data through
+ }
+ return rv;
+ }
+}
Modified: chukwa/trunk/src/packages/tarball/all.xml
URL: http://svn.apache.org/viewvc/chukwa/trunk/src/packages/tarball/all.xml?rev=1614808&r1=1614807&r2=1614808&view=diff
==============================================================================
--- chukwa/trunk/src/packages/tarball/all.xml (original)
+++ chukwa/trunk/src/packages/tarball/all.xml Thu Jul 31 04:04:59 2014
@@ -36,6 +36,10 @@
<outputDirectory>etc/chukwa</outputDirectory>
</fileSet>
<fileSet>
+ <directory>contrib/solr</directory>
+ <outputDirectory>etc/solr</outputDirectory>
+ </fileSet>
+ <fileSet>
<directory>bin</directory>
<includes>
<include>chukwa</include>
Added: chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java
URL: http://svn.apache.org/viewvc/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java (added)
+++ chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java Thu Jul 31 04:04:59 2014
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.chukwa.datacollection.writer.solr;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.chukwa.Chunk;
+import org.apache.hadoop.chukwa.ChunkImpl;
+import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.log4j.Logger;
+import org.apache.solr.SolrJettyTestBase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.core.CoreContainer;
+
+import junit.framework.Assert;
+
+public class TestSolrWriter extends SolrJettyTestBase {
+ private static Logger log = Logger.getLogger(TestSolrWriter.class);
+ CoreContainer container;
+
+ public void setUp() {
+ try {
+ String dataDir = System.getProperty("CHUKWA_DATA_DIR", "target/test/var");
+ container = new CoreContainer(dataDir);
+ container.load();
+
+ server = new EmbeddedSolrServer(container, "collection1" );
+ super.setUp();
+ } catch (Exception e) {
+ log.error(ExceptionUtil.getStackTrace(e));
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ public void tearDown() throws Exception {
+ if (server != null) {
+ server.shutdown();
+ }
+ super.tearDown();
+ }
+
+ /**
+ * Test adding a chunk to solr cloud, then query for the same chunk.
+ */
+ public void testCommit() {
+ ArrayList<Chunk> chunks = new ArrayList<Chunk>();
+ chunks.add(new ChunkImpl("Hadoop", "namenode",
+ System.currentTimeMillis(), "This is a test.".getBytes(), null));
+ try {
+ QueryResponse rsp = server.query(new SolrQuery("*:*"));
+ Assert.assertEquals(0, rsp.getResults().getNumFound());
+
+ SolrWriter sw = new SolrWriter();
+ sw.add(chunks);
+
+ // TODO: not a great way to test this - timing is easily out
+ // of whack due to parallel tests and various computer specs/load
+ Thread.sleep(1000); // wait 1 sec
+
+ // now check that it comes out...
+ rsp = server.query(new SolrQuery("data:test"));
+
+ int cnt = 0;
+ while (rsp.getResults().getNumFound() == 0) {
+ // wait and try again for slower/busier machines
+ // and/or parallel test effects.
+
+ if (cnt++ == 10) {
+ break;
+ }
+
+ Thread.sleep(2000); // wait 2 seconds...
+
+ rsp = server.query(new SolrQuery("data:test"));
+ }
+
+ Assert.assertEquals(1, rsp.getResults().getNumFound());
+
+ } catch (Exception e) {
+
+ }
+ }
+}