You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2015/03/16 05:40:24 UTC
svn commit: r1666856 [5/5] - in /lucene/dev/trunk/solr: ./
contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/
core/src/java/org/apache/solr/handler/
core/src/java/org/apache/solr/handler/component/
core/src/java/org/apache/solr/requ...
Added: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java?rev=1666856&view=auto
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java (added)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java Mon Mar 16 04:40:23 2015
@@ -0,0 +1,549 @@
+package org.apache.solr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.NoOpResponseParser;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.servlet.DirectSolrConnection;
+import org.noggit.JSONUtil;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+
+
+@SolrTestCaseJ4.SuppressSSL
+@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending","Asserting"})
+public class SolrTestCaseHS extends SolrTestCaseJ4 {
+
+ @SafeVarargs
+ public static <T> Set<T> set(T... a) {
+ LinkedHashSet<T> s = new LinkedHashSet<>();
+ for (T t : a) {
+ s.add(t);
+ }
+ return s;
+ }
+
+ public static <T> T rand(T... vals) {
+ return vals[ random().nextInt(vals.length) ];
+ }
+
+
+ public static ModifiableSolrParams params(SolrParams params, String... moreParams) {
+ ModifiableSolrParams msp = new ModifiableSolrParams(params);
+ for (int i=0; i<moreParams.length; i+=2) {
+ msp.add(moreParams[i], moreParams[i+1]);
+ }
+ return msp;
+ }
+
+ public static Map<String,Object> toObject(Doc doc, IndexSchema schema, Collection<String> fieldNames) {
+ Map<String,Object> result = new HashMap<>();
+ for (Fld fld : doc.fields) {
+ if (fieldNames != null && !fieldNames.contains(fld.ftype.fname)) continue;
+ SchemaField sf = schema.getField(fld.ftype.fname);
+ if (!sf.multiValued()) {
+ result.put(fld.ftype.fname, fld.vals.get(0));
+ } else {
+ result.put(fld.ftype.fname, fld.vals);
+ }
+ }
+ return result;
+ }
+
+
+ public static Object createDocObjects(Map<Comparable, Doc> fullModel, Comparator sort, int rows, Collection<String> fieldNames) {
+ List<Doc> docList = new ArrayList<>(fullModel.values());
+ Collections.sort(docList, sort);
+ List sortedDocs = new ArrayList(rows);
+ for (Doc doc : docList) {
+ if (sortedDocs.size() >= rows) break;
+ Map<String,Object> odoc = toObject(doc, h.getCore().getLatestSchema(), fieldNames);
+ sortedDocs.add(toObject(doc, h.getCore().getLatestSchema(), fieldNames));
+ }
+ return sortedDocs;
+ }
+
+
+ public static void compare(SolrQueryRequest req, String path, Object model, Map<Comparable, Doc> fullModel) throws Exception {
+ String strResponse = h.query(req);
+
+ Object realResponse = ObjectBuilder.fromJSON(strResponse);
+ String err = JSONTestUtil.matchObj(path, realResponse, model);
+ if (err != null) {
+ log.error("RESPONSE MISMATCH: " + err
+ + "\n\trequest="+req
+ + "\n\tresult="+strResponse
+ + "\n\texpected="+ JSONUtil.toJSON(model)
+ + "\n\tmodel="+ fullModel
+ );
+
+ // re-execute the request... good for putting a breakpoint here for debugging
+ String rsp = h.query(req);
+
+ fail(err);
+ }
+
+ }
+
+ /** Pass "null" for the client to query the local server */
+ public static void assertJQ(SolrClient client, SolrParams args, String... tests) throws Exception {
+ String resp;
+ resp = getJSON(client, args);
+ matchJSON(resp, tests);
+ }
+
+ public static void matchJSON(String response, String... tests) throws Exception {
+ boolean failed = false;
+
+ for (String test : tests) {
+ if (test == null || test.length()==0) continue;
+
+ try {
+ failed = true;
+ String err = JSONTestUtil.match(response, test, JSONTestUtil.DEFAULT_DELTA);
+ failed = false;
+ if (err != null) {
+ log.error("query failed JSON validation. error=" + err +
+ "\n expected =" + test +
+ "\n response = " + response
+ );
+ throw new RuntimeException(err);
+ }
+ } finally {
+ if (failed) {
+ log.error("JSON query validation threw an exception." +
+ "\n expected =" + test +
+ "\n response = " + response
+ );
+ }
+ }
+ }
+ }
+
+ /***
+ public static void clearNCache() {
+ SolrQueryRequest req = req();
+ req.getSearcher().getnCache().clear(); // OFF-HEAP
+ req.close();
+ }***/
+
+ public static void clearQueryCache() {
+ SolrQueryRequest req = req();
+ req.getSearcher();
+ req.close();
+ }
+
+
+ public static String getQueryResponse(SolrClient client, String wt, SolrParams params) throws Exception {
+ if (client == null) {
+ return getQueryResponse(wt, params);
+ }
+ ModifiableSolrParams p = new ModifiableSolrParams(params);
+ p.set("wt", wt);
+ String path = p.get("qt");
+ p.remove("qt");
+ p.set("indent","true");
+
+ QueryRequest query = new QueryRequest( p );
+ if (path != null) {
+ query.setPath(path);
+ }
+
+ query.setResponseParser(new NoOpResponseParser(wt));
+ NamedList<Object> rsp = client.request(query);
+
+ String raw = (String)rsp.get("response");
+
+ return raw;
+ }
+
+ public static String getQueryResponse(String wt, SolrParams params) throws Exception {
+ ModifiableSolrParams p = new ModifiableSolrParams(params);
+ p.set("wt", wt);
+ String path = p.get("qt");
+ p.remove("qt");
+ p.set("indent","true");
+
+ DirectSolrConnection connection = new DirectSolrConnection(h.getCore());
+ String raw = connection.request(path, p, null);
+ return raw;
+ }
+
+
+ public static String getJSON(SolrClient client, SolrParams params) throws Exception {
+ return getQueryResponse(client, "json", params);
+ }
+
+ /** Adds a document using the specific client, or to the local test core if null.
+ * Returns the version. TODO: work in progress... version not always returned. */
+ public static Long add(SolrClient client, SolrInputDocument sdoc, ModifiableSolrParams params) throws Exception {
+ if (client == null) {
+ Long version = addAndGetVersion( sdoc, params );
+ return version;
+ } else {
+ UpdateRequest updateRequest = new UpdateRequest();
+ if (params != null) {
+ updateRequest.setParams(params);
+ }
+ updateRequest.add( sdoc );
+ UpdateResponse rsp = updateRequest.process( client );
+ // TODO - return version
+ return null;
+ }
+ }
+
+
+
+ public static class Client {
+ ClientProvider provider;
+ ModifiableSolrParams queryDefaults;
+
+ public static Client localClient = new Client(null, 1);
+ public static Client localClient() {
+ return new Client(null, 1);
+ }
+
+ public Client(List<SolrClient> clients, int seed) {
+ if (clients != null) {
+ provider = new ClientProvider(clients, seed);
+ }
+ }
+
+ public static int hash(int x) {
+ // from Thomas Mueller
+ x = ((x >>> 16) ^ x) * 0x45d9f3b;
+ x = ((x >>> 16) ^ x) * 0x45d9f3b;
+ x = ((x >>> 16) ^ x);
+ return x;
+ }
+
+ public ModifiableSolrParams queryDefaults() {
+ if (queryDefaults == null) {
+ queryDefaults = new ModifiableSolrParams();
+ }
+ return queryDefaults;
+ }
+
+ public boolean local() {
+ return provider == null;
+ }
+
+ public void testJQ(SolrParams args, String... tests) throws Exception {
+ if (queryDefaults != null) {
+ ModifiableSolrParams newParams = params(queryDefaults);
+ newParams.add(args);
+ args = newParams;
+ }
+ SolrClient client = provider==null ? null : provider.client(null, args);
+ SolrTestCaseHS.assertJQ(client, args, tests);
+ }
+
+ public Long add(SolrInputDocument sdoc, ModifiableSolrParams params) throws Exception {
+ SolrClient client = provider==null ? null : provider.client(sdoc, params);
+ return SolrTestCaseHS.add(client, sdoc, params);
+ }
+
+ public void commit() throws IOException, SolrServerException {
+ if (local()) {
+ assertU(SolrTestCaseJ4.commit());
+ return;
+ }
+
+ for (SolrClient client : provider.all()) {
+ client.commit();
+ }
+ }
+
+ public void deleteByQuery(String query, ModifiableSolrParams params) throws IOException, SolrServerException {
+ if (local()) {
+ assertU(delQ(query)); // todo - handle extra params
+ return;
+ }
+
+ for (SolrClient client : provider.all()) {
+ client.deleteByQuery(query); // todo - handle extra params
+ }
+ }
+
+ }
+
+
+ public static class ClientProvider {
+ public static String idField = "id";
+
+ List<SolrClient> clients;
+ Random r;
+ int hashSeed;
+
+ // thisIsIgnored needed because we need a diff signature
+ public ClientProvider(List<SolrClient> clients, int seed) {
+ this.hashSeed = Client.hash(seed);
+ this.clients = clients;
+ r = new Random(seed);
+ }
+
+ public SolrClient client(SolrInputDocument sdoc, SolrParams params) {
+ String idStr = null;
+ if (sdoc != null) {
+ idStr = sdoc.getFieldValue(idField).toString();
+ } else if (params!=null) {
+ idStr = params.get(idField);
+ }
+
+ int hash;
+ if (idStr != null) {
+ // make the client chosen the same for a duplicate ID
+ hash = idStr.hashCode() ^ hashSeed;
+ } else {
+ hash = r.nextInt();
+ }
+
+
+ return clients.get( (hash & Integer.MAX_VALUE) % clients.size() );
+ }
+
+ public List<SolrClient> all() {
+ return clients;
+ }
+ }
+
+
+ //
+ // Helper to run an internal Jetty instance.
+ // Example:
+ // SolrInstance s1 = new SolrInstance(createTempDir("s1"), "solrconfig-tlog.xml", "schema_latest.xml");
+ // s1.start();
+ // SolrClient c1 = s1.getSolrJ();
+ // assertJQ(c1, params("q", "*:*"), "/response/numFound==3");
+ // String json = getJSON(c1, params("q","id:1"));
+ // s1.stop();
+ //
+ // To manage multiple servers, see SolrInstances
+ //
+ public static class SolrInstance {
+ private static Logger log = SolrTestCaseJ4.log;
+ private String collection = "collection1";
+ private int port = 0;
+ private String solrconfigFile;
+ private String schemaFile;
+ private File baseDir;
+ private JettySolrRunner jetty;
+ private SolrClient solrj;
+
+ private boolean homeCreated = false;
+
+
+ public SolrInstance(File homeDir, String solrconfigFile, String schemaFile) {
+ this.baseDir = homeDir;
+ this.solrconfigFile = solrconfigFile;
+ this.schemaFile = schemaFile;
+ }
+
+ public String getBaseDir() {
+ return baseDir.toString();
+ }
+
+ public String getBaseURL() {
+ return (SolrTestCaseJ4.isSSLMode() ? "https" : "http") + "://127.0.0.1:" + port + "/solr";
+ }
+
+ public String getCollectionURL() {
+ return getBaseURL() + "/" + collection;
+ }
+
+ /** string appropriate for passing in shards param (i.e. missing http://) */
+ public String getShardURL() {
+ return "127.0.0.1:" + port + "/solr" + "/" + collection;
+ }
+
+ public SolrClient getSolrJ() {
+ if (solrj == null) {
+ solrj = new HttpSolrClient(getCollectionURL());
+ }
+ return solrj;
+ }
+
+ /** If it needs to change */
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ public void createHome() throws Exception {
+ homeCreated=true;
+ SolrTestCaseJ4.copySolrHomeToTemp(baseDir, collection);
+ copyConfFile(baseDir, collection, solrconfigFile);
+ copyConfFile(baseDir, collection, schemaFile);
+
+ File collDir = new File(baseDir, collection);
+ try (Writer w = new OutputStreamWriter(Files.newOutputStream(collDir.toPath().resolve("core.properties")), Charsets.UTF_8)) {
+ Properties coreProps = new Properties();
+ coreProps.put("name", "collection1");
+ coreProps.put("config", solrconfigFile);
+ coreProps.put("schema", schemaFile);
+ coreProps.store(w, "");
+ }
+ }
+
+
+ public void start() throws Exception {
+ if (!homeCreated) {
+ createHome();
+ }
+
+ if (jetty == null) {
+ jetty = new JettySolrRunner(baseDir.getAbsolutePath(), "/solr", port, solrconfigFile, schemaFile, true, null, null, null);
+ }
+
+ // silly stuff included from solrconfig.snippet.randomindexconfig.xml
+ System.setProperty("solr.tests.maxBufferedDocs", String.valueOf(100000));
+
+ jetty.start();
+ port = jetty.getLocalPort();
+ log.info("===> Started solr server port=" + port + " home="+getBaseDir());
+ }
+
+ public void stop() throws Exception {
+ jetty.stop();
+ if (solrj != null) solrj.close();
+ }
+
+ public void tearDown() throws Exception {
+ IOUtils.deleteFilesIfExist(baseDir.toPath());
+ }
+
+ private static void copyConfFile(File dstRoot, String destCollection, String file) throws Exception {
+ File subHome = new File(dstRoot, destCollection + File.separator + "conf");
+ String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf";
+ FileUtils.copyFile(new File(top, file), new File(subHome, file));
+ }
+
+ public void copyConfigFile(File dstRoot, String destCollection, String file) throws Exception {
+ if (!homeCreated) {
+ createHome();
+ }
+
+ File subHome = new File(dstRoot, destCollection + File.separator + "conf");
+ String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf";
+ FileUtils.copyFile(new File(top, file), new File(subHome, file));
+ }
+
+ }
+
+
+ // Manages a number of Solr servers and provides a Client to partition documents and randomly assign query requests.
+ // Example:
+ // SolrInstances servers = new SolrInstances(3, "solrconfig-tlog.xml","schema_latest.xml");
+ // Client = servers.getClient(0);
+ // client.add(sdoc("id", "3"), null);
+ // client.commit();
+ // client.testJQ(params("q", "*:*"), "/response/numFound==3")
+ // servers.stop();
+ //
+ public static class SolrInstances {
+ public List<SolrInstance> slist;
+ public Client client;
+
+ public SolrInstances(int numServers, String solrconfig, String schema) throws Exception {
+ slist = new ArrayList<>(numServers);
+ for (int i=0; i<numServers; i++) {
+ SolrInstance instance = new SolrInstance(createTempDir("s"+ i).toFile(), solrconfig, schema);
+ slist.add(instance);
+ instance.start();
+ }
+ }
+
+ public void stop() throws Exception {
+ for (SolrInstance instance : slist) {
+ instance.stop();
+ }
+ }
+
+ // For params.set("shards", getShards())
+ public String getShards() {
+ return getShardsParam(slist);
+ }
+
+ public List<SolrClient> getSolrJs() {
+ List<SolrClient> solrjs = new ArrayList<>(slist.size());
+ for (SolrInstance instance : slist) {
+ solrjs.add( instance.getSolrJ() );
+ }
+ return solrjs;
+ }
+
+ public Client getClient(int seed) {
+ if (client == null) {
+ client = new Client(getSolrJs(), seed);
+ }
+ return client;
+ }
+
+ public static String getShardsParam(List<SolrInstance> instances) {
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (SolrInstance instance : instances) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(',');
+ }
+ sb.append( instance.getShardURL() );
+ }
+ return sb.toString();
+ }
+
+
+
+ }
+
+}