You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 09:56:15 UTC

[lucene] 19/42: Merge branch 'master' into jira/solr14155-1

This is an automated email from the ASF dual-hosted git repository.

dweiss pushed a commit to branch jira/solr14155-1
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit 6bc50e9ebfb76c291d5b369d708f93ec02355a34
Merge: eef0bf5 03d658a
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Thu Jul 16 16:36:52 2020 +1000

    Merge branch 'master' into jira/solr14155-1

 .gitignore                                         |   7 +-
 dev-tools/doap/lucene.rdf                          |   7 +
 dev-tools/doap/solr.rdf                            |   7 +
 lucene/CHANGES.txt                                 |  34 +++-
 .../lucene/index/TestBackwardsCompatibility.java   |   7 +-
 .../org/apache/lucene/index/index.8.6.0-cfs.zip    | Bin 0 -> 16442 bytes
 .../org/apache/lucene/index/index.8.6.0-nocfs.zip  | Bin 0 -> 16428 bytes
 .../test/org/apache/lucene/index/sorted.8.6.0.zip  | Bin 0 -> 140450 bytes
 .../lucene/queryparser/classic/FastCharStream.java |   9 +-
 .../flexible/standard/parser/FastCharStream.java   |  11 +-
 .../surround/parser/FastCharStream.java            |  11 +-
 solr/CHANGES.txt                                   |  23 ++-
 .../solr/handler/dataimport/ContextImpl.java       |   3 +-
 .../handler/dataimport/EntityProcessorWrapper.java |   4 +-
 .../handler/dataimport/XPathEntityProcessor.java   |   3 +-
 .../dataimport/TestFileListEntityProcessor.java    |   6 +-
 .../dataimport/TestLineEntityProcessor.java        |   6 +-
 .../solr/response/VelocityResponseWriter.java      |   2 -
 .../solr/cloud/ShardLeaderElectionContextBase.java |   4 +
 .../solr/cloud/api/collections/SplitShardCmd.java  |  11 +-
 .../solr/cloud/autoscaling/IndexSizeTrigger.java   |  89 ++++----
 .../solr/cloud/autoscaling/SearchRateTrigger.java  |  55 +++--
 .../cloud/autoscaling/sim/SimCloudManager.java     |   7 +-
 .../autoscaling/sim/SimClusterStateProvider.java   | 224 +++++++++++----------
 .../autoscaling/sim/SimNodeStateProvider.java      |  22 +-
 .../solr/cloud/autoscaling/sim/SimScenario.java    |   8 +-
 .../solr/cloud/autoscaling/sim/SimUtils.java       |  25 ++-
 .../autoscaling/sim/SnapshotCloudManager.java      |   4 +-
 .../autoscaling/sim/SnapshotNodeStateProvider.java |  66 +++---
 .../apache/solr/cloud/overseer/NodeMutator.java    |   4 +-
 .../apache/solr/cloud/overseer/ReplicaMutator.java |   4 +-
 .../src/java/org/apache/solr/core/SolrCore.java    |   6 +
 .../java/org/apache/solr/handler/IndexFetcher.java |  22 +-
 .../apache/solr/handler/ReplicationHandler.java    |   3 +-
 .../org/apache/solr/handler/SchemaHandler.java     |  46 ++++-
 .../solr/handler/admin/MetricsHistoryHandler.java  |   5 +-
 .../org/apache/solr/parser/FastCharStream.java     |  12 ++
 .../solr/pkg/PackageListeningClassLoader.java      |  16 +-
 .../java/org/apache/solr/response/XMLWriter.java   |  46 +++++
 .../apache/solr/schema/FieldTypePluginLoader.java  |  12 +-
 .../java/org/apache/solr/schema/IndexSchema.java   | 109 +++++-----
 .../org/apache/solr/schema/ManagedIndexSchema.java |   2 +-
 .../org/apache/solr/schema/PreAnalyzedField.java   |   2 +-
 .../solr/search/facet/FacetFieldProcessor.java     |   2 +-
 .../distance/GeoDistValueSourceParser.java         |   3 +-
 .../java/org/apache/solr/servlet/HttpSolrCall.java |   3 +-
 .../src/java/org/apache/solr/util/SolrCLI.java     |   7 +-
 .../solr/util/plugin/AbstractPluginLoader.java     |   8 +-
 .../test-files/runtimecode/schema-plugins.jar.bin  | Bin 0 -> 6814 bytes
 solr/core/src/test-files/runtimecode/sig.txt       |   5 +
 .../DistribDocExpirationUpdateProcessorTest.java   |   4 +-
 .../org/apache/solr/cloud/HttpPartitionTest.java   |   2 +-
 .../solr/cloud/MoveReplicaHDFSFailoverTest.java    |   4 +
 .../org/apache/solr/cloud/MoveReplicaHDFSTest.java |   4 +
 .../apache/solr/cloud/SaslZkACLProviderTest.java   |   4 +
 .../cloud/SharedFSAutoReplicaFailoverTest.java     |   5 +
 .../solr/cloud/TestSolrCloudWithKerberosAlt.java   |   4 +
 .../HdfsCollectionsAPIDistributedZkTest.java       |   4 +
 .../collections/TestHdfsCloudBackupRestore.java    |   5 +
 .../cloud/autoscaling/ComputePlanActionTest.java   |   3 +-
 .../HdfsAutoAddReplicasIntegrationTest.java        |   4 +
 .../SearchRateTriggerIntegrationTest.java          |   7 +-
 .../cloud/autoscaling/SearchRateTriggerTest.java   |  14 +-
 .../solr/cloud/autoscaling/TestPolicyCloud.java    |   9 +-
 .../autoscaling/sim/SimSolrCloudTestCase.java      |   9 +-
 .../sim/TestSimClusterStateProvider.java           |   3 +-
 .../cloud/autoscaling/sim/TestSimPolicyCloud.java  |   5 +-
 .../autoscaling/sim/TestSimTriggerIntegration.java |   8 +-
 .../autoscaling/sim/TestSnapshotCloudManager.java  |  19 +-
 .../solr/cloud/hdfs/HDFSCollectionsAPITest.java    |   4 +
 .../cloud/hdfs/HdfsBasicDistributedZk2Test.java    |   4 +
 .../cloud/hdfs/HdfsBasicDistributedZkTest.java     |   4 +
 .../hdfs/HdfsChaosMonkeyNothingIsSafeTest.java     |   4 +
 .../cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java  |   4 +
 .../apache/solr/cloud/hdfs/HdfsNNFailoverTest.java |   4 +
 .../solr/cloud/hdfs/HdfsRecoverLeaseTest.java      |   4 +
 .../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java |   4 +
 .../cloud/hdfs/HdfsRestartWhileUpdatingTest.java   |   4 +
 .../apache/solr/cloud/hdfs/HdfsSyncSliceTest.java  |   4 +
 .../apache/solr/cloud/hdfs/HdfsThreadLeakTest.java |   4 +
 .../HdfsTlogReplayBufferedWhileIndexingTest.java   |   4 +
 .../cloud/hdfs/HdfsUnloadDistributedZkTest.java    |   4 +
 .../hdfs/HdfsWriteToMultipleCollectionsTest.java   |   4 +
 .../org/apache/solr/cloud/hdfs/StressHdfsTest.java |   4 +
 .../org/apache/solr/cloud/rule/RuleEngineTest.java |   4 +-
 .../test/org/apache/solr/core/CoreSorterTest.java  |   2 +-
 .../apache/solr/core/HdfsDirectoryFactoryTest.java |   4 +
 .../solr/handler/TestHdfsBackupRestoreCore.java    |   4 +
 .../TestReplicationHandlerDiskOverFlow.java        |   2 +-
 .../handler/admin/MetricsHistoryHandlerTest.java   |   2 +-
 .../solr/handler/admin/ThreadDumpHandlerTest.java  | 169 ++++++++++++----
 .../apache/solr/index/hdfs/CheckHdfsIndexTest.java |   4 +
 .../src/test/org/apache/solr/pkg/TestPackages.java | 140 ++++++++++++-
 .../org/apache/solr/response/TestPushWriter.java   |  66 ++++--
 .../org/apache/solr/search/TestRecoveryHdfs.java   |   4 +
 .../solr/store/blockcache/BlockCacheTest.java      |  12 --
 .../apache/solr/store/hdfs/HdfsDirectoryTest.java  |   4 +
 .../solr/store/hdfs/HdfsLockFactoryTest.java       |   4 +
 .../org/apache/solr/update/TestHdfsUpdateLog.java  |   4 +
 .../org/apache/solr/util/TestCircuitBreaker.java   |  31 ++-
 solr/solr-ref-guide/src/_config.yml.template       |   1 +
 .../src/_includes/mathjax-support.html             |  22 ++
 solr/solr-ref-guide/src/_layouts/default.html      |   2 +
 solr/solr-ref-guide/src/aliases.adoc               |   9 +-
 solr/solr-ref-guide/src/enabling-ssl.adoc          |   2 +-
 .../src/kerberos-authentication-plugin.adoc        |   3 +-
 .../src/meta-docs/asciidoc-syntax.adoc             |  45 +++++
 .../src/the-query-elevation-component.adoc         |   2 +-
 solr/solr-ref-guide/src/using-solrj.adoc           |  23 +++
 .../solr/client/solrj/cloud/NodeStateProvider.java |   5 +-
 .../client/solrj/cloud/autoscaling/Clause.java     |   2 +-
 .../solrj/cloud/autoscaling/CoresVariable.java     |   6 +-
 .../autoscaling/DelegatingNodeStateProvider.java   |   3 +-
 .../solrj/cloud/autoscaling/FreeDiskVariable.java  |  31 +--
 .../cloud/autoscaling/MoveReplicaSuggester.java    |  13 +-
 .../solrj/cloud/autoscaling/PolicyHelper.java      |   9 +-
 .../solrj/cloud/autoscaling/ReplicaCount.java      |   6 +-
 .../solrj/cloud/autoscaling/ReplicaInfo.java       | 224 ---------------------
 .../solr/client/solrj/cloud/autoscaling/Row.java   |  47 +++--
 .../client/solrj/cloud/autoscaling/Suggester.java  |  14 +-
 .../client/solrj/cloud/autoscaling/Suggestion.java |   2 +-
 .../client/solrj/cloud/autoscaling/Variable.java   |   9 +-
 .../client/solrj/cloud/autoscaling/Violation.java  |   5 +-
 .../cloud/autoscaling/WithCollectionVariable.java  |   6 +-
 .../solrj/impl/SolrClientNodeStateProvider.java    |  30 +--
 .../solrj/request/JavaBinUpdateRequestCodec.java   |   2 +-
 .../java/org/apache/solr/common/SolrDocument.java  |   2 +
 .../org/apache/solr/common/SolrInputDocument.java  |   1 +
 .../solr/common/cloud/CompositeIdRouter.java       |   3 +-
 .../org/apache/solr/common/cloud/DocRouter.java    |   3 +-
 .../java/org/apache/solr/common/cloud/Replica.java | 189 ++++++++++++++---
 .../org/apache/solr/common/util/TextWriter.java    |  13 +-
 .../UsingSolrJRefGuideExamplesTest.java            |  54 ++++-
 .../solr/client/solrj/TestLBHttp2SolrClient.java   |   7 -
 .../solr/client/solrj/TestLBHttpSolrClient.java    |   7 -
 .../autoscaling/MoveReplicaSuggesterTest.java      |  32 +--
 .../client/solrj/cloud/autoscaling/TestPolicy.java |  98 ++++-----
 .../solrj/cloud/autoscaling/TestPolicy2.java       |  19 +-
 .../solr/client/solrj/request/TestCoreAdmin.java   |   8 -
 .../org/apache/solr/common/SolrDocumentTest.java   |   2 +-
 .../org/apache/solr/SolrIgnoredThreadsFilter.java  |   5 +
 .../src/java/org/apache/solr/SolrTestCase.java     |  10 +
 .../src/java/org/apache/solr/SolrTestCaseJ4.java   |   9 +-
 143 files changed, 1590 insertions(+), 1000 deletions(-)

diff --cc solr/core/src/java/org/apache/solr/core/SolrCore.java
index 714af89,4c9fae6..fb8a84f
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@@ -225,12 -224,11 +225,14 @@@ public final class SolrCore implements 
    private IndexReaderFactory indexReaderFactory;
    private final Codec codec;
    private final MemClassLoader memClassLoader;
+   //singleton listener for all packages used in schema
+   private final PackageListeningClassLoader schemaPluginsLoader;
  
 +  //a single package listener for all cores that require core reloading
 +  private final PackageListeningClassLoader coreReloadingClassLoader;
 +  //singleton listener for all packages used in schema
 +  private final PackageListeningClassLoader schemaPluginsLoader;
    private final CircuitBreakerManager circuitBreakerManager;
 -
    private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
  
    private final ReentrantLock ruleExpiryLock;
@@@ -957,11 -956,7 +962,12 @@@
                () -> setLatestSchema(configSet.getIndexSchema(true)));
        this.packageListeners.addListener(schemaPluginsLoader);
        IndexSchema schema = configSet.getIndexSchema();
+ 
 +      this.coreReloadingClassLoader = new PackageListeningClassLoader(coreContainer,
 +              resourceLoader,
 +              solrConfig::maxPackageVersion,
 +              () -> coreContainer.reload(name, uniqueId));
 +      this.packageListeners.addListener(coreReloadingClassLoader);
        this.configSetProperties = configSet.getProperties();
        // Initialize the metrics manager
        this.coreMetricManager = initCoreMetricManager(solrConfig);
diff --cc solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
index 6b66d84,ced4bd0..5e8bf69
--- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java
@@@ -37,54 -37,42 +37,52 @@@ import java.util.function.Function
   * */
  public class PackageListeningClassLoader implements SolrClassLoader , PackageListeners.Listener {
      private final CoreContainer coreContainer;
 -    private final SolrResourceLoader coreResourceLoader;
++
 +    private final SolrResourceLoader fallbackResourceLoader;
      private final Function<String, String> pkgVersionSupplier;
      /** package name and the versions that we are tracking
       */
      private Map<String ,PackageAPI.PkgVersion> packageVersions =  new HashMap<>(1);
      private final Runnable onReload;
  
++
 +    /**
 +     * @param fallbackResourceLoader The {@link SolrResourceLoader} to use if no package is specified
 +     * @param pkgVersionSupplier Get the version configured for a given package
 +     * @param onReload The callback function that should be run if a package is updated
 +     */
      public PackageListeningClassLoader(CoreContainer coreContainer,
 -                                       SolrResourceLoader coreResourceLoader,
 +                                       SolrResourceLoader fallbackResourceLoader,
                                         Function<String, String> pkgVersionSupplier,
                                         Runnable onReload) {
          this.coreContainer = coreContainer;
 -        this.coreResourceLoader = coreResourceLoader;
 +        this.fallbackResourceLoader = fallbackResourceLoader;
          this.pkgVersionSupplier = pkgVersionSupplier;
          this.onReload = () -> {
 -            packageVersions = new HashMap<>();
 -            onReload.run();
 -        };
 -    }
 +            //clear our current versions. When new classes are loaded from packages, fresh data can be populated
-             packageVersions = new HashMap<>();
-             onReload.run();
-         };
-     }
++
  
  
      @Override
      public <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {
          PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
--        if(cName.pkg == null){
 -            return coreResourceLoader.newInstance(cname, expectedType, subpackages);
++        if(cName.pkg == null) {
 +            return fallbackResourceLoader.newInstance(cname, expectedType, subpackages);
          } else {
-             PackageLoader.Package.Version version = findPackageVersion(cName, true);
 -            PackageLoader.Package.Version version = findPkgVersion(cName);
--            return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages));
++            PackageLoader.Package.Version version = findPackageVersion(cName, true);            return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subpackages));
  
          }
      }
  
 -    private PackageLoader.Package.Version findPkgVersion(PluginInfo.ClassName cName) {
++
 +    /**
 +     * This looks up for package and also listens for that package if required
 +     * @param cName The class name
 +     */
 +    public PackageLoader.Package.Version findPackageVersion(PluginInfo.ClassName cName, boolean registerListener) {
          PackageLoader.Package.Version theVersion = coreContainer.getPackageLoader().getPackage(cName.pkg).getLatest(pkgVersionSupplier.apply(cName.pkg));
 -        packageVersions.put(cName.pkg, theVersion.getPkgVersion());
 +        if(registerListener) {
 +            packageVersions.put(cName.pkg, theVersion.getPkgVersion());
-         }
          return theVersion;
      }
  
@@@ -113,9 -101,9 +111,10 @@@
      public <T> T newInstance(String cname, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {
          PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
          if (cName.pkg == null) {
 -            return coreResourceLoader.newInstance(cname, expectedType, subPackages, params, args);
 +            return fallbackResourceLoader.newInstance(cname, expectedType, subPackages, params, args);
          } else {
 -            PackageLoader.Package.Version version = findPkgVersion(cName);
 +            PackageLoader.Package.Version version = findPackageVersion(cName, true);
++
              return applyResourceLoaderAware(version, version.getLoader().newInstance(cName.className, expectedType, subPackages, params, args));
          }
      }
@@@ -124,11 -112,11 +123,10 @@@
      public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
          PluginInfo.ClassName cName = new PluginInfo.ClassName(cname);
          if (cName.pkg == null) {
 -            return coreResourceLoader.findClass(cname, expectedType);
 +            return fallbackResourceLoader.findClass(cname, expectedType);
          } else {
 -            PackageLoader.Package.Version version = findPkgVersion(cName);
 +            PackageLoader.Package.Version version = findPackageVersion(cName, true);
              return version.getLoader().findClass(cName.className, expectedType);
--
          }
      }