You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/08/06 17:59:04 UTC
[06/33] lucenenet git commit: Ported Lucene.Net.Benchmark + tests
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
new file mode 100644
index 0000000..9be1b5d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/TaskSequence.cs
@@ -0,0 +1,662 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Stats;
+using Lucene.Net.Support.Threading;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Text;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Sequence of parallel or sequential tasks.
+ /// </summary>
+ public class TaskSequence : PerfTask
+ {
+ public static int REPEAT_EXHAUST = -2;
+ private IList<PerfTask> tasks;
+ private int repetitions = 1;
+ private bool parallel;
+ private TaskSequence parent;
+ private bool letChildReport = true;
+ private int rate = 0;
+ private bool perMin = false; // rate, if set, is, by default, be sec.
+ private string seqName;
+ private bool exhausted = false;
+ private bool resetExhausted = false;
+ private PerfTask[] tasksArray;
+ private bool anyExhaustibleTasks;
+ private bool collapsable = false; // to not collapse external sequence named in alg.
+
+ private bool fixedTime; // true if we run for fixed time
+ private double runTimeSec; // how long to run for
+ private readonly long logByTimeMsec;
+
+ public TaskSequence(PerfRunData runData, String name, TaskSequence parent, bool parallel)
+ : base(runData)
+ {
+ collapsable = (name == null);
+ name = (name != null ? name : (parallel ? "Par" : "Seq"));
+ SetName(name);
+ SetSequenceName();
+ this.parent = parent;
+ this.parallel = parallel;
+ tasks = new List<PerfTask>();
+ logByTimeMsec = runData.Config.Get("report.time.step.msec", 0);
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ InitTasksArray();
+ for (int i = 0; i < tasksArray.Length; i++)
+ {
+ tasksArray[i].Dispose();
+ }
+ RunData.DocMaker.Dispose();
+ }
+ }
+
+ private void InitTasksArray()
+ {
+ if (tasksArray == null)
+ {
+ int numTasks = tasks.Count;
+ tasksArray = new PerfTask[numTasks];
+ for (int k = 0; k < numTasks; k++)
+ {
+ tasksArray[k] = tasks[k];
+ anyExhaustibleTasks |= tasksArray[k] is ResetInputsTask;
+ anyExhaustibleTasks |= tasksArray[k] is TaskSequence;
+ }
+ }
+ if (!parallel && logByTimeMsec != 0 && !letChildReport)
+ {
+ countsByTime = new int[1];
+ }
+ }
+
+ /// <summary>
+ /// Gets the parallel.
+ /// </summary>
+ public virtual bool IsParallel
+ {
+ get { return parallel; }
+ }
+
+ /// <summary>
+ /// Gets the repetitions.
+ /// </summary>
+ public virtual int Repetitions
+ {
+ get { return repetitions; }
+ }
+
+ private int[] countsByTime;
+
+ public virtual void SetRunTime(double sec)
+ {
+ runTimeSec = sec;
+ fixedTime = true;
+ }
+
+ /// <summary>
+ /// Sets the repetitions.
+ /// </summary>
+ /// <param name="repetitions">The repetitions to set.</param>
+ public virtual void SetRepetitions(int repetitions)
+ {
+ fixedTime = false;
+ this.repetitions = repetitions;
+ if (repetitions == REPEAT_EXHAUST)
+ {
+ if (IsParallel)
+ {
+ throw new Exception("REPEAT_EXHAUST is not allowed for parallel tasks");
+ }
+ }
+ SetSequenceName();
+ }
+
+ /// <summary>
+ /// Gets the parent.
+ /// </summary>
+ public virtual TaskSequence Parent
+ {
+ get { return parent; }
+ }
+
+ /// <seealso cref="PerfTask.DoLogic()"/>
+ public override int DoLogic()
+ {
+ exhausted = resetExhausted = false;
+ return (parallel ? DoParallelTasks() : DoSerialTasks());
+ }
+
+ private class RunBackgroundTask : ThreadClass
+ {
+ private readonly PerfTask task;
+ private readonly bool letChildReport;
+ private volatile int count;
+
+ public RunBackgroundTask(PerfTask task, bool letChildReport)
+ {
+ this.task = task;
+ this.letChildReport = letChildReport;
+ }
+
+ public virtual void StopNow()
+ {
+ task.StopNow();
+ }
+
+ public virtual int Count
+ {
+ get { return count; }
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ count = task.RunAndMaybeStats(letChildReport);
+ }
+ catch (Exception e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ }
+ }
+
+ private int DoSerialTasks()
+ {
+ if (rate > 0)
+ {
+ return DoSerialTasksWithRate();
+ }
+
+ InitTasksArray();
+ int count = 0;
+
+ long runTime = (long)(runTimeSec * 1000);
+ List<RunBackgroundTask> bgTasks = null;
+
+ long t0 = Support.Time.CurrentTimeMilliseconds();
+ for (int k = 0; fixedTime || (repetitions == REPEAT_EXHAUST && !exhausted) || k < repetitions; k++)
+ {
+ if (Stop)
+ {
+ break;
+ }
+ for (int l = 0; l < tasksArray.Length; l++)
+ {
+ PerfTask task = tasksArray[l];
+ if (task.RunInBackground)
+ {
+ if (bgTasks == null)
+ {
+ bgTasks = new List<RunBackgroundTask>();
+ }
+ RunBackgroundTask bgTask = new RunBackgroundTask(task, letChildReport);
+ bgTask.Priority = (task.BackgroundDeltaPriority + Thread.CurrentThread.Priority);
+ bgTask.Start();
+ bgTasks.Add(bgTask);
+ }
+ else
+ {
+ try
+ {
+ int inc = task.RunAndMaybeStats(letChildReport);
+ count += inc;
+ if (countsByTime != null)
+ {
+ int slot = (int)((Support.Time.CurrentTimeMilliseconds() - t0) / logByTimeMsec);
+ if (slot >= countsByTime.Length)
+ {
+ countsByTime = ArrayUtil.Grow(countsByTime, 1 + slot);
+ }
+ countsByTime[slot] += inc;
+ }
+ if (anyExhaustibleTasks)
+ UpdateExhausted(task);
+ }
+ catch (NoMoreDataException /*e*/)
+ {
+ exhausted = true;
+ }
+ }
+ }
+ if (fixedTime && Support.Time.CurrentTimeMilliseconds() - t0 > runTime)
+ {
+ repetitions = k + 1;
+ break;
+ }
+ }
+
+ if (bgTasks != null)
+ {
+ foreach (RunBackgroundTask bgTask in bgTasks)
+ {
+ bgTask.StopNow();
+ }
+ foreach (RunBackgroundTask bgTask in bgTasks)
+ {
+ bgTask.Join();
+ count += bgTask.Count;
+ }
+ }
+
+ if (countsByTime != null)
+ {
+ RunData.Points.CurrentStats.SetCountsByTime(countsByTime, logByTimeMsec);
+ }
+
+ Stop = false;
+
+ return count;
+ }
+
+ private int DoSerialTasksWithRate()
+ {
+ InitTasksArray();
+ long delayStep = (perMin ? 60000 : 1000) / rate;
+ long nextStartTime = Support.Time.CurrentTimeMilliseconds();
+ int count = 0;
+ long t0 = Support.Time.CurrentTimeMilliseconds();
+ for (int k = 0; (repetitions == REPEAT_EXHAUST && !exhausted) || k < repetitions; k++)
+ {
+ if (Stop)
+ {
+ break;
+ }
+ for (int l = 0; l < tasksArray.Length; l++)
+ {
+ PerfTask task = tasksArray[l];
+ while (!Stop)
+ {
+ long waitMore = nextStartTime - Support.Time.CurrentTimeMilliseconds();
+ if (waitMore > 0)
+ {
+ // TODO: better to use condition to notify
+ Thread.Sleep(1);
+ }
+ else
+ {
+ break;
+ }
+ }
+ if (Stop)
+ {
+ break;
+ }
+ nextStartTime += delayStep; // this aims at avarage rate.
+ try
+ {
+ int inc = task.RunAndMaybeStats(letChildReport);
+ count += inc;
+ if (countsByTime != null)
+ {
+ int slot = (int)((Support.Time.CurrentTimeMilliseconds() - t0) / logByTimeMsec);
+ if (slot >= countsByTime.Length)
+ {
+ countsByTime = ArrayUtil.Grow(countsByTime, 1 + slot);
+ }
+ countsByTime[slot] += inc;
+ }
+
+ if (anyExhaustibleTasks)
+ UpdateExhausted(task);
+ }
+ catch (NoMoreDataException /*e*/)
+ {
+ exhausted = true;
+ }
+ }
+ }
+ Stop = false;
+ return count;
+ }
+
+ // update state regarding exhaustion.
+ private void UpdateExhausted(PerfTask task)
+ {
+ if (task is ResetInputsTask)
+ {
+ exhausted = false;
+ resetExhausted = true;
+ }
+ else if (task is TaskSequence)
+ {
+ TaskSequence t = (TaskSequence)task;
+ if (t.resetExhausted)
+ {
+ exhausted = false;
+ resetExhausted = true;
+ t.resetExhausted = false;
+ }
+ else
+ {
+ exhausted |= t.exhausted;
+ }
+ }
+ }
+
+ private class ParallelTask : ThreadClass
+ {
+ private int count;
+ private readonly PerfTask task;
+ private readonly TaskSequence outerInstance;
+
+ // LUCENENET specific - expose field through property
+ public int Count
+ {
+ get { return count; }
+ }
+
+ // LUCENENET specific - expose field through property
+ public PerfTask Task
+ {
+ get { return task; }
+ }
+
+ public ParallelTask(TaskSequence outerInstance, PerfTask task)
+ {
+ this.outerInstance = outerInstance;
+ this.task = task;
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ int n = task.RunAndMaybeStats(outerInstance.letChildReport);
+ if (outerInstance.anyExhaustibleTasks)
+ {
+ outerInstance.UpdateExhausted(task);
+ }
+ count += n;
+ }
+ catch (NoMoreDataException)
+ {
+ outerInstance.exhausted = true;
+ }
+ catch (Exception e)
+ {
+ throw new Exception(e.ToString(), e);
+ }
+ }
+ }
+
+ public override void StopNow()
+ {
+ base.StopNow();
+ // Forwards top request to children
+ if (runningParallelTasks != null)
+ {
+ foreach (ParallelTask t in runningParallelTasks)
+ {
+ if (t != null)
+ {
+ t.Task.StopNow();
+ }
+ }
+ }
+ }
+
+ ParallelTask[] runningParallelTasks;
+
+ private int DoParallelTasks()
+ {
+
+ TaskStats stats = RunData.Points.CurrentStats;
+
+ InitTasksArray();
+ ParallelTask[] t = runningParallelTasks = new ParallelTask[repetitions * tasks.Count];
+ // prepare threads
+ int index = 0;
+ for (int k = 0; k < repetitions; k++)
+ {
+ for (int i = 0; i < tasksArray.Length; i++)
+ {
+ PerfTask task = (PerfTask)(tasksArray[i].Clone());
+ t[index++] = new ParallelTask(this, task);
+ }
+ }
+ // run threads
+ StartThreads(t);
+
+ if (Stop)
+ {
+ foreach (ParallelTask task in t)
+ {
+ task.Task.StopNow();
+ }
+ }
+
+ // wait for all threads to complete
+ int count = 0;
+ for (int i = 0; i < t.Length; i++)
+ {
+ t[i].Join();
+ count += t[i].Count;
+ if (t[i].Task is TaskSequence)
+ {
+ TaskSequence sub = (TaskSequence)t[i].Task;
+ if (sub.countsByTime != null)
+ {
+ if (countsByTime == null)
+ {
+ countsByTime = new int[sub.countsByTime.Length];
+ }
+ else if (countsByTime.Length < sub.countsByTime.Length)
+ {
+ countsByTime = ArrayUtil.Grow(countsByTime, sub.countsByTime.Length);
+ }
+ for (int j = 0; j < sub.countsByTime.Length; j++)
+ {
+ countsByTime[j] += sub.countsByTime[j];
+ }
+ }
+ }
+ }
+
+ if (countsByTime != null)
+ {
+ stats.SetCountsByTime(countsByTime, logByTimeMsec);
+ }
+
+ // return total count
+ return count;
+ }
+
+ // run threads
+ private void StartThreads(ParallelTask[] t)
+ {
+ if (rate > 0)
+ {
+ StartlThreadsWithRate(t);
+ return;
+ }
+ for (int i = 0; i < t.Length; i++)
+ {
+ t[i].Start();
+ }
+ }
+
+ // run threads with rate
+ private void StartlThreadsWithRate(ParallelTask[] t)
+ {
+ long delayStep = (perMin ? 60000 : 1000) / rate;
+ long nextStartTime = Support.Time.CurrentTimeMilliseconds();
+ for (int i = 0; i < t.Length; i++)
+ {
+ long waitMore = nextStartTime - Support.Time.CurrentTimeMilliseconds();
+ if (waitMore > 0)
+ {
+ Thread.Sleep((int)waitMore);
+ }
+ nextStartTime += delayStep; // this aims at average rate of starting threads.
+ t[i].Start();
+ }
+ }
+
+ public virtual void AddTask(PerfTask task)
+ {
+ tasks.Add(task);
+ task.Depth = Depth + 1;
+ }
+
+ /// <seealso cref="object.ToString()"/>
+ public override string ToString()
+ {
+ string padd = GetPadding();
+ StringBuilder sb = new StringBuilder(base.ToString());
+ sb.Append(parallel ? " [" : " {");
+ sb.Append(NEW_LINE);
+ foreach (PerfTask task in tasks)
+ {
+ sb.Append(task.ToString());
+ sb.Append(NEW_LINE);
+ }
+ sb.Append(padd);
+ sb.Append(!letChildReport ? ">" : (parallel ? "]" : "}"));
+ if (fixedTime)
+ {
+ sb.AppendFormat(CultureInfo.InvariantCulture, " {0:N}s", runTimeSec);
+ }
+ else if (repetitions > 1)
+ {
+ sb.Append(" * " + repetitions);
+ }
+ else if (repetitions == REPEAT_EXHAUST)
+ {
+ sb.Append(" * EXHAUST");
+ }
+ if (rate > 0)
+ {
+ sb.Append(", rate: " + rate + "/" + (perMin ? "min" : "sec"));
+ }
+ if (RunInBackground)
+ {
+ sb.Append(" &");
+ int x = BackgroundDeltaPriority;
+ if (x != 0)
+ {
+ sb.Append(x);
+ }
+ }
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Execute child tasks in a way that they do not report their time separately.
+ /// </summary>
+ public virtual void SetNoChildReport()
+ {
+ letChildReport = false;
+ foreach (PerfTask task in tasks)
+ {
+ if (task is TaskSequence)
+ {
+ ((TaskSequence)task).SetNoChildReport();
+ }
+ }
+ }
+
+ /// <summary>
+ /// Returns the rate per minute: how many operations should be performed in a minute.
+ /// If 0 this has no effect.
+ /// </summary>
+ /// <returns>The rate per min: how many operations should be performed in a minute.</returns>
+ public virtual int GetRate()
+ {
+ return (perMin ? rate : 60 * rate);
+ }
+
+ /// <summary>
+ ///
+ /// </summary>
+ /// <param name="rate">The rate to set.</param>
+ /// <param name="perMin"></param>
+ public virtual void SetRate(int rate, bool perMin)
+ {
+ this.rate = rate;
+ this.perMin = perMin;
+ SetSequenceName();
+ }
+
+ private void SetSequenceName()
+ {
+ seqName = base.GetName();
+ if (repetitions == REPEAT_EXHAUST)
+ {
+ seqName += "_Exhaust";
+ }
+ else if (repetitions > 1)
+ {
+ seqName += "_" + repetitions;
+ }
+ if (rate > 0)
+ {
+ seqName += "_" + rate + (perMin ? "/min" : "/sec");
+ }
+ if (parallel && seqName.ToLowerInvariant().IndexOf("par") < 0)
+ {
+ seqName += "_Par";
+ }
+ }
+
+ public override string GetName()
+ {
+ return seqName; // override to include more info
+ }
+
+ /// <summary>
+ /// Gets the tasks.
+ /// </summary>
+ public virtual IList<PerfTask> Tasks
+ {
+ get { return tasks; }
+ }
+
+ /// <seealso cref="ICloneable.Clone()"/>
+ public override object Clone()
+ {
+ TaskSequence res = (TaskSequence)base.Clone();
+ res.tasks = new List<PerfTask>();
+ for (int i = 0; i < tasks.Count; i++)
+ {
+ res.tasks.Add((PerfTask)tasks[i].Clone());
+ }
+ return res;
+ }
+
+ /// <summary>
+ /// Return <c>true</c> if can be collapsed in case it is outermost sequence.
+ /// </summary>
+ public virtual bool IsCollapsable
+ {
+ get { return collapsable; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs
new file mode 100644
index 0000000..f69ce2b
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/UpdateDocTask.cs
@@ -0,0 +1,99 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using System;
+using System.Globalization;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Update a document, using <see cref="IndexWriter.UpdateDocument(Term, System.Collections.Generic.IEnumerable{IIndexableField})"/>,
+ /// optionally with of a certain size.
+ /// <para/>
+ /// Other side effects: none.
+ /// <para/>
+ /// Takes optional param: document size.
+ /// </summary>
+ public class UpdateDocTask : PerfTask
+ {
+ public UpdateDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ private int docSize = 0;
+
+ // volatile data passed between setup(), doLogic(), tearDown().
+ private Document doc = null;
+
+ public override void Setup()
+ {
+ base.Setup();
+ DocMaker docMaker = RunData.DocMaker;
+ if (docSize > 0)
+ {
+ doc = docMaker.MakeDocument(docSize);
+ }
+ else
+ {
+ doc = docMaker.MakeDocument();
+ }
+ }
+
+ public override void TearDown()
+ {
+ doc = null;
+ base.TearDown();
+ }
+
+ public override int DoLogic()
+ {
+ string docID = doc.Get(DocMaker.ID_FIELD);
+ if (docID == null)
+ {
+ throw new InvalidOperationException("document must define the docid field");
+ }
+ IndexWriter iw = RunData.IndexWriter;
+ iw.UpdateDocument(new Term(DocMaker.ID_FIELD, docID), doc);
+ return 1;
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ return "updated " + recsCount + " docs";
+ }
+
+ /// <summary>
+ /// Set the params (docSize only)
+ /// </summary>
+ /// <param name="params">docSize, or 0 for no limit.</param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ docSize = (int)float.Parse(@params, CultureInfo.InvariantCulture);
+ }
+
+ /// <seealso cref="PerfTask.SupportsParams"/>
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs
new file mode 100644
index 0000000..7dad964
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitForMergesTask.cs
@@ -0,0 +1,36 @@
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Waits for merges to finish.
+ /// </summary>
+ public class WaitForMergesTask : PerfTask
+ {
+ public WaitForMergesTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override int DoLogic()
+ {
+ RunData.IndexWriter.WaitForMerges();
+ return 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs
new file mode 100644
index 0000000..67f648d
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WaitTask.cs
@@ -0,0 +1,89 @@
+using System;
+using System.Globalization;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Simply waits for the specified (via the parameter) amount
+ /// of time. For example Wait(30s) waits for 30 seconds.
+ /// This is useful with background tasks to control how long
+ /// the tasks run.
+ /// <para/>
+ /// You can specify h, m, or s (hours, minutes, seconds) as
+ /// the trailing time unit. No unit is interpreted as
+ /// seconds.
+ /// </summary>
+ public class WaitTask : PerfTask
+ {
+ private double waitTimeSec;
+
+ public WaitTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ if (@params != null)
+ {
+ int multiplier;
+ if (@params.EndsWith("s", StringComparison.Ordinal))
+ {
+ multiplier = 1;
+ @params = @params.Substring(0, @params.Length - 1);
+ }
+ else if (@params.EndsWith("m", StringComparison.Ordinal))
+ {
+ multiplier = 60;
+ @params = @params.Substring(0, @params.Length - 1);
+ }
+ else if (@params.EndsWith("h", StringComparison.Ordinal))
+ {
+ multiplier = 3600;
+ @params = @params.Substring(0, @params.Length - 1);
+ }
+ else
+ {
+ // Assume seconds
+ multiplier = 1;
+ }
+
+ waitTimeSec = double.Parse(@params, CultureInfo.InvariantCulture) * multiplier;
+ }
+ else
+ {
+ throw new ArgumentException("you must specify the wait time, eg: 10.0s, 4.5m, 2h");
+ }
+ }
+
+ public override int DoLogic()
+ {
+ Thread.Sleep((int)(1000 * waitTimeSec));
+ return 0;
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs
new file mode 100644
index 0000000..3e1f8d8
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WarmTask.cs
@@ -0,0 +1,64 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Warm reader task: retrieve all reader documents.
+ /// </summary>
+ /// <remarks>
+ /// Note: This task reuses the reader if it is already open.
+ /// Otherwise a reader is opened at start and closed at the end.
+ /// <para/>
+ /// Other side effects: counts additional 1 (record) for each
+ /// retrieved (non null) document.
+ /// </remarks>
+ public class WarmTask : ReadTask
+ {
+ public WarmTask(PerfRunData runData)
+ : base(runData)
+ {
+ }
+
+ public override bool WithRetrieve
+ {
+ get { return false; }
+ }
+
+ public override bool WithSearch
+ {
+ get { return false; }
+ }
+
+ public override bool WithTraverse
+ {
+ get { return false; }
+ }
+
+ public override bool WithWarm
+ {
+ get { return true; }
+ }
+
+ public override IQueryMaker GetQueryMaker()
+ {
+ return null; // not required for this task.
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs
new file mode 100644
index 0000000..d70836e
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteEnwikiLineDocTask.cs
@@ -0,0 +1,72 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using System;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A <see cref="WriteLineDocTask"/> which for Wikipedia input, will write category pages
+ /// to another file, while remaining pages will be written to the original file.
+ /// The categories file is derived from the original file, by adding a prefix "categories-".
+ /// </summary>
+ public class WriteEnwikiLineDocTask : WriteLineDocTask
+ {
+ private readonly TextWriter categoryLineFileOut;
+
+ public WriteEnwikiLineDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ Stream @out = StreamUtils.GetOutputStream(CategoriesLineFile(new FileInfo(m_fname)));
+ categoryLineFileOut = new StreamWriter(@out, Encoding.UTF8);
+ WriteHeader(categoryLineFileOut);
+ }
+
+ /// <summary>Compose categories line file out of original line file</summary>
+ public static FileInfo CategoriesLineFile(FileInfo f)
+ {
+ DirectoryInfo dir = f.Directory;
+ string categoriesName = "categories-" + f.Name;
+ return dir == null ? new FileInfo(categoriesName) : new FileInfo(System.IO.Path.Combine(dir.FullName, categoriesName));
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ categoryLineFileOut.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ protected override TextWriter LineFileOut(Document doc)
+ {
+ IIndexableField titleField = doc.GetField(DocMaker.TITLE_FIELD);
+ if (titleField != null && titleField.GetStringValue().StartsWith("Category:", StringComparison.Ordinal))
+ {
+ return categoryLineFileOut;
+ }
+ return base.LineFileOut(doc);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
new file mode 100644
index 0000000..f9e7546
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/WriteLineDocTask.cs
@@ -0,0 +1,238 @@
+using Lucene.Net.Benchmarks.ByTask.Feeds;
+using Lucene.Net.Benchmarks.ByTask.Utils;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+
+namespace Lucene.Net.Benchmarks.ByTask.Tasks
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A task which writes documents, one line per document. Each line is in the
+ /// following format: title <TAB> date <TAB> body. The output of this
+ /// task can be consumed by <see cref="LineDocSource"/> and is intended
+ /// to save the IO overhead of opening a file per document to be indexed.
+ /// </summary>
+ /// <remarks>
+ /// The format of the output is set according to the output file extension.
+ /// Compression is recommended when the output file is expected to be large.
+ /// See info on file extensions in <see cref="FileType"/>.
+ /// <para/>
+ /// Supports the following parameters:
+ /// <list type="bullet">
+ /// <item><term>line.file.out</term><description>the name of the file to write the output to. That parameter is mandatory. <b>NOTE:</b> the file is re-created.</description></item>
+ /// <item><term>line.fields</term><description>which fields should be written in each line. (optional, default: <see cref="DEFAULT_FIELDS"/>).</description></item>
+ /// <item><term>sufficient.fields</term><description>
+ /// list of field names, separated by comma, which,
+ /// if all of them are missing, the document will be skipped. For example, to require
+ /// that at least one of f1,f2 is not empty, specify: "f1,f2" in this field. To specify
+ /// that no field is required, i.e. that even empty docs should be emitted, specify <b>","</b>
+ /// (optional, default: <see cref="DEFAULT_SUFFICIENT_FIELDS"/>).
+ /// </description></item>
+ /// </list>
+ /// <para/>
+ /// <b>NOTE:</b> this class is not thread-safe and if used by multiple threads the
+ /// output is unspecified (as all will write to the same output file in a
+ /// non-synchronized way).
+ /// </remarks>
+ public class WriteLineDocTask : PerfTask
+ {
+ public static readonly string FIELDS_HEADER_INDICATOR = "FIELDS_HEADER_INDICATOR###";
+
+ public readonly static char SEP = '\t';
+
+ /// <summary>
+ /// Fields to be written by default
+ /// </summary>
+ public static readonly string[] DEFAULT_FIELDS = new string[] {
+ DocMaker.TITLE_FIELD,
+ DocMaker.DATE_FIELD,
+ DocMaker.BODY_FIELD,
+ };
+
+ /// <summary>
+ /// Default fields which at least one of them is required to not skip the doc.
+ /// </summary>
+ public static readonly string DEFAULT_SUFFICIENT_FIELDS = DocMaker.TITLE_FIELD + ',' + DocMaker.BODY_FIELD;
+
+ private int docSize = 0;
+ protected readonly string m_fname;
+ private readonly TextWriter lineFileOut;
+ private readonly DocMaker docMaker;
+ private readonly ThreadLocal<StringBuilder> threadBuffer = new ThreadLocal<StringBuilder>();
+ private readonly ThreadLocal<Regex> threadNormalizer = new ThreadLocal<Regex>();
+ private readonly string[] fieldsToWrite;
+ private readonly bool[] sufficientFields;
+ private readonly bool checkSufficientFields;
+
+
+ public WriteLineDocTask(PerfRunData runData)
+ : base(runData)
+ {
+ Config config = runData.Config;
+ m_fname = config.Get("line.file.out", null);
+ if (m_fname == null)
+ {
+ throw new ArgumentException("line.file.out must be set");
+ }
+ Stream @out = StreamUtils.GetOutputStream(new FileInfo(m_fname));
+ lineFileOut = new StreamWriter(@out, Encoding.UTF8);
+ docMaker = runData.DocMaker;
+
+ // init fields
+ string f2r = config.Get("line.fields", null);
+ if (f2r == null)
+ {
+ fieldsToWrite = DEFAULT_FIELDS;
+ }
+ else
+ {
+ if (f2r.IndexOf(SEP) >= 0)
+ {
+ throw new ArgumentException("line.fields " + f2r + " should not contain the separator char: " + SEP);
+ }
+ fieldsToWrite = f2r.Split(',').TrimEnd();
+ }
+
+ // init sufficient fields
+ sufficientFields = new bool[fieldsToWrite.Length];
+ string suff = config.Get("sufficient.fields", DEFAULT_SUFFICIENT_FIELDS);
+ if (",".Equals(suff))
+ {
+ checkSufficientFields = false;
+ }
+ else
+ {
+ checkSufficientFields = true;
+ HashSet<string> sf = new HashSet<string>(suff.Split(new char[] { ',' }).TrimEnd());
+ for (int i = 0; i < fieldsToWrite.Length; i++)
+ {
+ if (sf.Contains(fieldsToWrite[i]))
+ {
+ sufficientFields[i] = true;
+ }
+ }
+ }
+
+ WriteHeader(lineFileOut);
+ }
+
+ /// <summary>
+ /// Write header to the lines file - indicating how to read the file later.
+ /// </summary>
+ protected virtual void WriteHeader(TextWriter @out)
+ {
+ StringBuilder sb = threadBuffer.Value;
+ if (sb == null)
+ {
+ sb = new StringBuilder();
+ threadBuffer.Value = sb;
+ }
+ sb.Length = 0;
+ sb.Append(FIELDS_HEADER_INDICATOR);
+ foreach (string f in fieldsToWrite)
+ {
+ sb.Append(SEP).Append(f);
+ }
+ @out.WriteLine(sb.ToString());
+ }
+
+ protected override string GetLogMessage(int recsCount)
+ {
+ return "Wrote " + recsCount + " line docs";
+ }
+
+ public override int DoLogic()
+ {
+ Document doc = docSize > 0 ? docMaker.MakeDocument(docSize) : docMaker.MakeDocument();
+
+ Regex matcher = threadNormalizer.Value;
+ if (matcher == null)
+ {
+ matcher = new Regex("[\t\r\n]+");
+ threadNormalizer.Value = matcher;
+ }
+
+ StringBuilder sb = threadBuffer.Value;
+ if (sb == null)
+ {
+ sb = new StringBuilder();
+ threadBuffer.Value = sb;
+ }
+ sb.Length = 0;
+
+ bool sufficient = !checkSufficientFields;
+ for (int i = 0; i < fieldsToWrite.Length; i++)
+ {
+ IIndexableField f = doc.GetField(fieldsToWrite[i]);
+ string text = f == null ? "" : matcher.Replace(f.GetStringValue(), " ").Trim();
+ sb.Append(text).Append(SEP);
+ sufficient |= text.Length > 0 && sufficientFields[i];
+ }
+ if (sufficient)
+ {
+ sb.Length = sb.Length - 1; // remove redundant last separator
+ // lineFileOut is a PrintWriter, which synchronizes internally in println.
+ LineFileOut(doc).WriteLine(sb.ToString());
+ }
+
+ return 1;
+ }
+
+ /// <summary>
+ /// Selects output line file by written doc.
+ /// Default: original output line file.
+ /// </summary>
+ protected virtual TextWriter LineFileOut(Document doc)
+ {
+ return lineFileOut;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ lineFileOut.Dispose();
+ }
+ base.Dispose(disposing);
+ }
+
+ /// <summary>
+ /// Set the params (docSize only)
+ /// </summary>
+ /// <param name="params">docSize, or 0 for no limit.</param>
+ public override void SetParams(string @params)
+ {
+ base.SetParams(@params);
+ docSize = (int)float.Parse(@params, CultureInfo.InvariantCulture);
+ }
+
+ public override bool SupportsParams
+ {
+ get { return true; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
new file mode 100644
index 0000000..7e6523e
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/Algorithm.cs
@@ -0,0 +1,459 @@
+using Lucene.Net.Benchmarks.ByTask.Tasks;
+using Lucene.Net.Support;
+using Lucene.Net.Support.IO;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Test algorithm, as read from file
+ /// </summary>
+ public class Algorithm
+ {
+ private TaskSequence sequence;
+ private readonly string[] taskPackages;
+
+ /// <summary>
+ /// Read algorithm from file.
+ /// Property examined: alt.tasks.packages == comma separated list of
+ /// alternate Assembly names where tasks would be searched for, when not found
+ /// in the default Assembly (that of <see cref="PerfTask"/>).
+ /// If the same task class appears in more than one Assembly, the Assembly
+ /// indicated first in this list will be used.
+ /// <para/>
+ /// The Lucene.Net implementation differs from Lucene in that all
+ /// referenced assemblies are also scanned for the type. However,
+ /// alt.tasks.packages may be included for assemblies that are
+ /// not referenced in your project.
+ /// </summary>
+ /// <param name="runData">perf-run-data used at running the tasks.</param>
+ /// <exception cref="Exception">if errors while parsing the algorithm.</exception>
+ public Algorithm(PerfRunData runData)
+ {
+ Config config = runData.Config;
+ taskPackages = InitTasksPackages(config);
+ string algTxt = config.AlgorithmText;
+ sequence = new TaskSequence(runData, null, null, false);
+ TaskSequence currSequence = sequence;
+ PerfTask prevTask = null;
+ StreamTokenizer stok = new StreamTokenizer(new StringReader(algTxt));
+ stok.CommentChar('#');
+ stok.IsEOLSignificant = false;
+ stok.QuoteChar('"');
+ stok.QuoteChar('\'');
+ stok.OrdinaryChar('/');
+ stok.OrdinaryChar('(');
+ stok.OrdinaryChar(')');
+ bool colonOk = false;
+ bool isDisableCountNextTask = false; // only for primitive tasks
+ currSequence.Depth = 0;
+
+ while (stok.NextToken() != StreamTokenizer.TT_EOF)
+ {
+ switch (stok.TokenType)
+ {
+
+ case StreamTokenizer.TT_WORD:
+ string s = stok.StringValue;
+ PerfTask task = (PerfTask)Activator.CreateInstance(TaskClass(config, s), runData);
+ task.AlgLineNum = stok.LineNumber;
+ task.DisableCounting = isDisableCountNextTask;
+ isDisableCountNextTask = false;
+ currSequence.AddTask(task);
+ if (task is RepSumByPrefTask)
+ {
+ stok.NextToken();
+ string prefix = stok.StringValue;
+ if (prefix == null || prefix.Length == 0)
+ {
+ throw new Exception("named report prefix problem - " + stok.ToString());
+ }
+ ((RepSumByPrefTask)task).SetPrefix(prefix);
+ }
+ // check for task param: '(' someParam ')'
+ stok.NextToken();
+ if (stok.TokenType != '(')
+ {
+ stok.PushBack();
+ }
+ else
+ {
+ // get params, for tasks that supports them - allow recursive parenthetical expressions
+ stok.IsEOLSignificant = true; // Allow params tokenizer to keep track of line number
+ StringBuilder @params = new StringBuilder();
+ stok.NextToken();
+ if (stok.TokenType != ')')
+ {
+ int count = 1;
+ while (true)
+ {
+ switch (stok.TokenType)
+ {
+ case StreamTokenizer.TT_NUMBER:
+ {
+ @params.Append(stok.NumberValue);
+ break;
+ }
+ case StreamTokenizer.TT_WORD:
+ {
+ @params.Append(stok.StringValue);
+ break;
+ }
+ case StreamTokenizer.TT_EOF:
+ {
+ throw new Exception("Unexpexted EOF: - " + stok.ToString());
+ }
+ case '"':
+ case '\'':
+ {
+ @params.Append((char)stok.TokenType);
+ // re-escape delimiters, if any
+ @params.Append(stok.StringValue.Replace("" + (char)stok.TokenType, @"\" + (char)stok.TokenType));
+ @params.Append((char)stok.TokenType);
+ break;
+ }
+ case '(':
+ {
+ @params.Append((char)stok.TokenType);
+ ++count;
+ break;
+ }
+ case ')':
+ {
+ if (--count >= 1)
+ { // exclude final closing parenthesis
+ @params.Append((char)stok.TokenType);
+ }
+ else
+ {
+ goto BALANCED_PARENS_BREAK;
+ }
+ break;
+ }
+ default:
+ {
+ @params.Append((char)stok.TokenType);
+ break;
+ }
+ }
+ stok.NextToken();
+ }
+ BALANCED_PARENS_BREAK: { }
+ }
+ stok.IsEOLSignificant = false;
+ string prm = @params.ToString().Trim();
+ if (prm.Length > 0)
+ {
+ task.SetParams(prm);
+ }
+ }
+
+ // ---------------------------------------
+ colonOk = false; prevTask = task;
+ break;
+
+ default:
+ char c = (char)stok.TokenType;
+
+ switch (c)
+ {
+
+ case ':':
+ if (!colonOk) throw new Exception("colon unexpexted: - " + stok.ToString());
+ colonOk = false;
+ // get repetitions number
+ stok.NextToken();
+ if ((char)stok.TokenType == '*')
+ {
+ ((TaskSequence)prevTask).SetRepetitions(TaskSequence.REPEAT_EXHAUST);
+ }
+ else
+ {
+ if (stok.TokenType != StreamTokenizer.TT_NUMBER)
+ {
+ throw new Exception("expected repetitions number or XXXs: - " + stok.ToString());
+ }
+ else
+ {
+ double num = stok.NumberValue;
+ stok.NextToken();
+ if (stok.TokenType == StreamTokenizer.TT_WORD && stok.StringValue.Equals("s", StringComparison.Ordinal))
+ {
+ ((TaskSequence)prevTask).SetRunTime(num);
+ }
+ else
+ {
+ stok.PushBack();
+ ((TaskSequence)prevTask).SetRepetitions((int)num);
+ }
+ }
+ }
+ // check for rate specification (ops/min)
+ stok.NextToken();
+ if (stok.TokenType != ':')
+ {
+ stok.PushBack();
+ }
+ else
+ {
+ // get rate number
+ stok.NextToken();
+ if (stok.TokenType != StreamTokenizer.TT_NUMBER) throw new Exception("expected rate number: - " + stok.ToString());
+ // check for unit - min or sec, sec is default
+ stok.NextToken();
+ if (stok.TokenType != '/')
+ {
+ stok.PushBack();
+ ((TaskSequence)prevTask).SetRate((int)stok.NumberValue, false); // set rate per sec
+ }
+ else
+ {
+ stok.NextToken();
+ if (stok.TokenType != StreamTokenizer.TT_WORD) throw new Exception("expected rate unit: 'min' or 'sec' - " + stok.ToString());
+ string unit = stok.StringValue.ToLowerInvariant();
+ if ("min".Equals(unit, StringComparison.Ordinal))
+ {
+ ((TaskSequence)prevTask).SetRate((int)stok.NumberValue, true); // set rate per min
+ }
+ else if ("sec".Equals(unit, StringComparison.Ordinal))
+ {
+ ((TaskSequence)prevTask).SetRate((int)stok.NumberValue, false); // set rate per sec
+ }
+ else
+ {
+ throw new Exception("expected rate unit: 'min' or 'sec' - " + stok.ToString());
+ }
+ }
+ }
+ colonOk = false;
+ break;
+
+ case '{':
+ case '[':
+ // a sequence
+ // check for sequence name
+ string name = null;
+ stok.NextToken();
+ if (stok.TokenType != '"')
+ {
+ stok.PushBack();
+ }
+ else
+ {
+ name = stok.StringValue;
+ if (stok.TokenType != '"' || name == null || name.Length == 0)
+ {
+ throw new Exception("sequence name problem - " + stok.ToString());
+ }
+ }
+ // start the sequence
+ TaskSequence seq2 = new TaskSequence(runData, name, currSequence, c == '[');
+ currSequence.AddTask(seq2);
+ currSequence = seq2;
+ colonOk = false;
+ break;
+
+ case '&':
+ if (currSequence.IsParallel)
+ {
+ throw new Exception("Can only create background tasks within a serial task");
+ }
+ stok.NextToken();
+ int deltaPri;
+ if (stok.TokenType != StreamTokenizer.TT_NUMBER)
+ {
+ stok.PushBack();
+ deltaPri = 0;
+ }
+ else
+ {
+ // priority
+ deltaPri = (int)stok.NumberValue;
+ }
+
+ if (prevTask == null)
+ {
+ throw new Exception("& was unexpected");
+ }
+ else if (prevTask.RunInBackground)
+ {
+ throw new Exception("double & was unexpected");
+ }
+ else
+ {
+ prevTask.SetRunInBackground(deltaPri);
+ }
+ break;
+
+ case '>':
+ currSequence.SetNoChildReport(); /* intentional fallthrough */
+ // end sequence
+ colonOk = true; prevTask = currSequence;
+ currSequence = currSequence.Parent;
+ break;
+ case '}':
+ case ']':
+ // end sequence
+ colonOk = true; prevTask = currSequence;
+ currSequence = currSequence.Parent;
+ break;
+
+ case '-':
+ isDisableCountNextTask = true;
+ break;
+
+ } //switch(c)
+ break;
+
+ } //switch(stok.ttype)
+
+ }
+
+ if (sequence != currSequence)
+ {
+ throw new Exception("Unmatched sequences");
+ }
+
+ // remove redundant top level enclosing sequences
+ while (sequence.IsCollapsable && sequence.Repetitions == 1 && sequence.GetRate() == 0)
+ {
+ IList<PerfTask> t = sequence.Tasks;
+ if (t != null && t.Count == 1)
+ {
+ PerfTask p = t[0];
+ if (p is TaskSequence)
+ {
+ sequence = (TaskSequence)p;
+ continue;
+ }
+ }
+ break;
+ }
+ }
+
+ private string[] InitTasksPackages(Config config)
+ {
+ // LUCENENET specific - changing the logic a bit
+ // to add all referenced assemblies by default.
+ // The alt.tasks.packages parameter still exists, but
+ // it is only necessary for assemblies that are not
+ // referenced by the host assembly.
+
+ HashSet<string> result = new HashSet<string>();
+ string alts = config.Get("alt.tasks.packages", null);
+ string dfltPkg = typeof(PerfTask).GetTypeInfo().Assembly.GetName().Name;
+ string[] referencedAssemblies = AssemblyUtils.GetReferencedAssemblies().Select(a => a.GetName().Name).ToArray();
+ result.Add(dfltPkg);
+
+ if (alts == null)
+ {
+ result.UnionWith(referencedAssemblies);
+ return result.ToArray();
+ }
+
+ foreach (string alt in alts.Split(',').TrimEnd())
+ {
+ result.Add(alt);
+ }
+ result.UnionWith(referencedAssemblies);
+ return result.ToArray();
+ }
+
+ private Type TaskClass(Config config, string taskName)
+ {
+ foreach (string pkg in taskPackages)
+ {
+ Type result = LoadType(pkg, taskName + "Task");
+ if (result != null)
+ {
+ return result;
+ }
+ }
+ // can only get here if failed to instantiate
+ throw new TypeLoadException(taskName + " not found in packages " + Collections.ToString(taskPackages));
+ }
+
+ private Type LoadType(string assemblyName, string typeName)
+ {
+ return Assembly.Load(assemblyName).DefinedTypes.FirstOrDefault(t => t.Name == typeName);
+ }
+
+ public override string ToString()
+ {
+ string newline = Environment.NewLine;
+ StringBuilder sb = new StringBuilder();
+ sb.Append(sequence.ToString());
+ sb.Append(newline);
+ return sb.ToString();
+ }
+
+ /// <summary>
+ /// Execute this algorithm.
+ /// </summary>
+ public virtual void Execute()
+ {
+ try
+ {
+ sequence.RunAndMaybeStats(true);
+ }
+ finally
+ {
+ sequence.Dispose();
+ }
+ }
+
+ /// <summary>
+ /// Expert: for test purposes, return all tasks participating in this algorithm.
+ /// </summary>
+ /// <returns>All tasks participating in this algorithm.</returns>
+ public virtual IList<PerfTask> ExtractTasks()
+ {
+ List<PerfTask> res = new List<PerfTask>();
+ ExtractTasks(res, sequence);
+ return res;
+ }
+
+ private void ExtractTasks(IList<PerfTask> extrct, TaskSequence seq)
+ {
+ if (seq == null)
+ return;
+ extrct.Add(seq);
+ IList<PerfTask> t = sequence.Tasks;
+ if (t == null)
+ return;
+ foreach (PerfTask p in t)
+ {
+ if (p is TaskSequence)
+ {
+ ExtractTasks(extrct, (TaskSequence)p);
+ }
+ else
+ {
+ extrct.Add(p);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b515271d/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs b/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs
new file mode 100644
index 0000000..63c17cb
--- /dev/null
+++ b/src/Lucene.Net.Benchmark/ByTask/Utils/AnalyzerFactory.cs
@@ -0,0 +1,156 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Util;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Benchmarks.ByTask.Utils
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// A factory to create an analyzer.
+ /// </summary>
+ /// <seealso cref="Tasks.AnalyzerFactoryTask"/>
+ public sealed class AnalyzerFactory
+ {
+ private IList<CharFilterFactory> charFilterFactories;
+ private TokenizerFactory tokenizerFactory;
+ private IList<TokenFilterFactory> tokenFilterFactories;
+ private string name = null;
+ private int? positionIncrementGap = null;
+ private int? offsetGap = null;
+
+ public AnalyzerFactory(IList<CharFilterFactory> charFilterFactories,
+ TokenizerFactory tokenizerFactory,
+ IList<TokenFilterFactory> tokenFilterFactories)
+ {
+ this.charFilterFactories = charFilterFactories;
+ Debug.Assert(null != tokenizerFactory);
+ this.tokenizerFactory = tokenizerFactory;
+ this.tokenFilterFactories = tokenFilterFactories;
+ }
+
+ // LUCENENET TODO: Properties ?
+ public void SetName(string name)
+ {
+ this.name = name;
+ }
+
+ public void SetPositionIncrementGap(int positionIncrementGap) // LUCENENET TODO: Nullable?
+ {
+ this.positionIncrementGap = positionIncrementGap;
+ }
+
+ public void SetOffsetGap(int offsetGap) // LUCENENET TODO: Nullable?
+ {
+ this.offsetGap = offsetGap;
+ }
+
+ public Analyzer Create()
+ {
+ return new AnalyzerAnonymousHelper(this);
+ }
+
+ private class AnalyzerAnonymousHelper : Analyzer
+ {
+ private readonly AnalyzerFactory outerInstance;
+
+ public AnalyzerAnonymousHelper(AnalyzerFactory outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected override TextReader InitReader(string fieldName, TextReader reader)
+ {
+ if (outerInstance.charFilterFactories != null && outerInstance.charFilterFactories.Count > 0)
+ {
+ TextReader wrappedReader = reader;
+ foreach (CharFilterFactory charFilterFactory in outerInstance.charFilterFactories)
+ {
+ wrappedReader = charFilterFactory.Create(wrappedReader);
+ }
+ reader = wrappedReader;
+ }
+ return reader;
+ }
+
+ protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+ {
+ Tokenizer tokenizer = outerInstance.tokenizerFactory.Create(reader);
+ TokenStream tokenStream = tokenizer;
+ foreach (TokenFilterFactory filterFactory in outerInstance.tokenFilterFactories)
+ {
+ tokenStream = filterFactory.Create(tokenStream);
+ }
+ return new TokenStreamComponents(tokenizer, tokenStream);
+ }
+
+ public override int GetPositionIncrementGap(string fieldName)
+ {
+ return outerInstance.positionIncrementGap.HasValue
+ ? outerInstance.positionIncrementGap.Value
+ : base.GetPositionIncrementGap(fieldName);
+ }
+
+ public override int GetOffsetGap(string fieldName)
+ {
+ return outerInstance.offsetGap.HasValue
+ ? outerInstance.offsetGap.Value
+ : base.GetOffsetGap(fieldName);
+ }
+ }
+
+ public override string ToString()
+ {
+ StringBuilder sb = new StringBuilder("AnalyzerFactory(");
+ if (null != name)
+ {
+ sb.Append("name:");
+ sb.Append(name);
+ sb.Append(", ");
+ }
+ if (null != positionIncrementGap)
+ {
+ sb.Append("positionIncrementGap:");
+ sb.Append(positionIncrementGap);
+ sb.Append(", ");
+ }
+ if (null != offsetGap)
+ {
+ sb.Append("offsetGap:");
+ sb.Append(offsetGap);
+ sb.Append(", ");
+ }
+ foreach (CharFilterFactory charFilterFactory in charFilterFactories)
+ {
+ sb.Append(charFilterFactory);
+ sb.Append(", ");
+ }
+ sb.Append(tokenizerFactory);
+ foreach (TokenFilterFactory tokenFilterFactory in tokenFilterFactories)
+ {
+ sb.Append(", ");
+ sb.Append(tokenFilterFactory);
+ }
+ sb.Append(')');
+ return sb.ToString();
+ }
+ }
+}