You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by sl...@apache.org on 2021/02/26 15:07:10 UTC
[daffodil] branch master updated: Add debugger "info suspensions"
and "set diffExcludes" commands
This is an automated email from the ASF dual-hosted git repository.
slawrence pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/daffodil.git
The following commit(s) were added to refs/heads/master by this push:
new 66c272b Add debugger "info suspensions" and "set diffExcludes" commands
66c272b is described below
commit 66c272b7c21117f9f4b7ed2b986ce6c1dea72102
Author: Steve Lawrence <sl...@apache.org>
AuthorDate: Wed Feb 24 13:39:38 2021 -0500
Add debugger "info suspensions" and "set diffExcludes" commands
- Add new trait for info commands that are Seq's rather than simple
values, implementing act() and diff() functions
- Use this new trait to create a new "info suspensions" command to list
the current suspensions. For example:
(debug) info suspensions
suspensions:
SuppressableSeparatorUnparserSuspendableOperation for title
RegionSplitSuspendableOperation for title
SuppressableSeparatorUnparserSuspendableOperation for title
(debug) info diff
suspensions:
+ SuppressableSeparatorUnparserSuspendableOperation for title
- Add a new "set diffExcludes" command to exclude info commands from
showing up in "info diff". This can be useful if there are certain
diffs that happen all the time that the user doesn't care about. For
example:
(debug) set diffExcludes bitPosition bitLimit suspensions
- Skip debug steps during suspensions. This currently causes an abort
(some suspension state should never be accessed), but also can cause
confusing during debugging, especially related to diffs, since
Daffodil temporarily time travel when evaluating suspensions. More
thought is needed to figure out best how to debug suspensions so the
time travel aspect is obvious and understandable.
DAFFODIL-758
---
.../org/apache/daffodil/CLI/input/input1.txt.xml | 25 +++++
.../org/apache/daffodil/CLI/input/input9.txt.xml | 32 +++++++
.../it/scala/org/apache/daffodil/CLI/Util.scala | 8 ++
.../apache/daffodil/debugger/TestCLIDebugger.scala | 86 ++++++++++++++++-
.../daffodil/debugger/InteractiveDebugger.scala | 102 ++++++++++++++++++++-
.../daffodil/processors/ProcessorStateBases.scala | 6 +-
.../daffodil/processors/SuspensionTracker.scala | 2 +
.../daffodil/processors/parsers/PState.scala | 2 +
.../daffodil/processors/unparsers/UState.scala | 4 +
9 files changed, 259 insertions(+), 8 deletions(-)
diff --git a/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input1.txt.xml b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input1.txt.xml
new file mode 100644
index 0000000..021110b
--- /dev/null
+++ b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input1.txt.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<tns:matrix xmlns:tns="http://www.example.org/example1/">
+ <tns:row>
+ <tns:cell>0</tns:cell>
+ <tns:cell>1</tns:cell>
+ <tns:cell>2</tns:cell>
+ </tns:row>
+</tns:matrix>
diff --git a/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input9.txt.xml b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input9.txt.xml
new file mode 100644
index 0000000..77a6ae5
--- /dev/null
+++ b/daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input9.txt.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ex:list xmlns:ex="http://example.com">
+ <ex:Item>
+ <ex:description>Shirts</ex:description>
+ <ex:comment>Sold on Monday</ex:comment>
+ <ex:quantity>30</ex:quantity>
+ <ex:price>17.99</ex:price>
+ </ex:Item>
+ <ex:Item>
+ <ex:description>Shoes</ex:description>
+ <ex:comment>Sold on Tuesday</ex:comment>
+ <ex:quantity>23</ex:quantity>
+ <ex:price>89.99</ex:price>
+ </ex:Item>
+</ex:list>
diff --git a/daffodil-cli/src/it/scala/org/apache/daffodil/CLI/Util.scala b/daffodil-cli/src/it/scala/org/apache/daffodil/CLI/Util.scala
index f943bb2..ab2b415 100644
--- a/daffodil-cli/src/it/scala/org/apache/daffodil/CLI/Util.scala
+++ b/daffodil-cli/src/it/scala/org/apache/daffodil/CLI/Util.scala
@@ -150,6 +150,14 @@ object Util {
}
}
+ def devNull(): String = {
+ if (isWindows) {
+ "NUL"
+ } else {
+ "/dev/null"
+ }
+ }
+
def makeMultipleCmds(cmds: Array[String]): String = {
if (isWindows) {
cmds.mkString(" & ")
diff --git a/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala b/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala
index 3ff7f00..7202392 100644
--- a/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala
+++ b/daffodil-cli/src/it/scala/org/apache/daffodil/debugger/TestCLIDebugger.scala
@@ -18,11 +18,14 @@
package org.apache.daffodil.debugger
import org.junit.Test
-import org.apache.daffodil.CLI.Util
-import net.sf.expectit.matcher.Matchers.contains
+
import net.sf.expectit.matcher.Matchers.allOf
+import net.sf.expectit.matcher.Matchers.contains
+import net.sf.expectit.matcher.Matchers.regexp
import net.sf.expectit.matcher.Matchers.times
+import org.apache.daffodil.CLI.Util
+
class TestCLIdebugger {
val DAFFODIL_JAVA_OPTS = Map("DAFFODIL_JAVA_OPTS" -> "-Xms256m -Xmx2048m -Djline.terminal=jline.UnsupportedTerminal -Dfile.encoding=UTF-8")
@@ -1320,4 +1323,83 @@ class TestCLIdebugger {
}
}
+ @Test def test_CLI_Debugger_info_diff_04(): Unit = {
+ val schemaFile = Util.daffodilPath("daffodil-test/src/test/resources/org/apache/daffodil/section06/entities/charClassEntities.dfdl.xsd")
+ val inputFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input1.txt.xml")
+ val (testSchemaFile, testInputFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile), Util.cmdConvert(inputFile)) else (schemaFile, inputFile)
+
+ val shell = if (Util.isWindows) Util.start("", envp = DAFFODIL_JAVA_OPTS) else Util.start("")
+
+ try {
+ val cmd = String.format("%s -d unparse -s %s -r matrix -o %s %s", Util.binPath, testSchemaFile, Util.devNull, testInputFile)
+ shell.sendLine(cmd)
+ shell.expect(contains("(debug)"))
+ shell.sendLine("display info diff")
+ shell.expect(contains("(debug)"))
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.expect(contains("bitPosition: 0 -> 8"))
+ shell.sendLine("set diffExcludes childIndex")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.expect(regexp("\\+ Suppressable.* for cell"))
+ shell.sendLine("step")
+ shell.expect(regexp("\\+ Alignment.* for cell"))
+ shell.sendLine("info suspensions")
+ shell.expect(regexp("Suppressable.* for cell"))
+ shell.expect(regexp("Alignment.* for cell"))
+ shell.sendLine("quit")
+ } finally {
+ shell.close()
+ }
+ }
+
+ @Test def test_CLI_Debugger_info_diff_05(): Unit = {
+ val schemaFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/cli_schema_03.dfdl.xsd")
+ val inputFile = Util.daffodilPath("daffodil-cli/src/it/resources/org/apache/daffodil/CLI/input/input9.txt.xml")
+ val (testSchemaFile, testInputFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile), Util.cmdConvert(inputFile)) else (schemaFile, inputFile)
+
+ val shell = if (Util.isWindows) Util.start("", envp = DAFFODIL_JAVA_OPTS) else Util.start("")
+
+ try {
+ val cmd = String.format("%s -d unparse -r list -s %s -o %s %s", Util.binPath, testSchemaFile, Util.devNull, testInputFile)
+ shell.sendLine(cmd)
+ shell.expect(contains("(debug)"))
+ shell.sendLine("set diffExcludes doesNotExist1 bitLimit doesNotExist2")
+ shell.expect(contains("unknown or undiffable info commands: doesNotExist1, doesNotExist2"))
+
+ shell.sendLine("display info diff")
+ shell.sendLine("break Item")
+ shell.sendLine("continue")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.expect(regexp("\\+ SuppressableSeparator.* ex:Item"))
+ shell.sendLine("continue")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.expect(regexp("\\+ RegionSplit.* ex:Item"))
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.expect(regexp("\\- RegionSplit.* ex:Item"))
+ shell.sendLine("info suspensions")
+ shell.expect(regexp("SuppressableSeparator.* ex:Item"))
+ shell.sendLine("step")
+ shell.sendLine("step")
+ shell.sendLine("quit")
+ } finally {
+ shell.close()
+ }
+ }
+
+
}
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala
index 0e55ea7..7ba02d4 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/debugger/InteractiveDebugger.scala
@@ -43,6 +43,7 @@ import org.apache.daffodil.processors._
import org.apache.daffodil.processors.parsers.ConvertTextCombinatorParser
import org.apache.daffodil.processors.parsers._
import org.apache.daffodil.processors.unparsers.UState
+import org.apache.daffodil.processors.unparsers.UStateForSuspension
import org.apache.daffodil.processors.unparsers.Unparser
import org.apache.daffodil.schema.annotation.props.gen.Representation
import org.apache.daffodil.util.DPathUtil
@@ -118,6 +119,9 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
/* whether to remove hidden elements when displaying the infoset */
var removeHidden: Boolean = false
+ /* list of info commands to exclude when running 'info diff' */
+ var diffExcludes: Seq[String] = Seq.empty
+
/* stores the last actual command (i.e. not a "") that was executed */
var lastCommand: String = ""
@@ -158,6 +162,11 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
}
def debugStep(state: ParseOrUnparseState, processor: Processor, ignoreBreakpoints: Boolean): Unit = {
+ // ignore debug steps called during suspensions, those must be handled differently
+ if (state.isInstanceOf[UStateForSuspension]) {
+ return
+ }
+
debugState = debugState match {
case _ if ((state.processorStatus ne Success) && DebuggerConfig.breakOnFailure) => DebugState.Pause
case DebugState.Continue | DebugState.Trace if !ignoreBreakpoints => {
@@ -1197,6 +1206,7 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
InfoPath,
InfoParser,
InfoPointsOfUncertainty,
+ InfoSuspensions,
InfoUnparser,
InfoVariables)
@@ -1329,6 +1339,37 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
}
}
+ trait InfoSeqValue[A] extends InfoDiffable { self: DebugCommand =>
+
+ def getSeqValue(state: StateForDebugger): Seq[A]
+
+ def act(args: Seq[String], state: ParseOrUnparseState, processor: Processor): DebugState.Type = {
+ debugPrintln("%s:".format(self.name))
+ getSeqValue(state).foreach { value =>
+ debugPrintln("%s".format(value.toString), " ")
+ }
+ DebugState.Pause
+ }
+
+ def diff(pre: StateForDebugger, post: StateForDebugger): Boolean = {
+ val preSeq = getSeqValue(pre)
+ val postSeq = getSeqValue(post)
+ val removed = preSeq.diff(postSeq)
+ val added = postSeq.diff(preSeq)
+ val hasDiff = removed.nonEmpty || added.nonEmpty
+ if (hasDiff) {
+ debugPrintln("%s:".format(self.name), " ")
+ removed.foreach { v =>
+ debugPrintln("- %s".format(v.toString), " ")
+ }
+ added.foreach { v =>
+ debugPrintln("+ %s".format(v.toString), " ")
+ }
+ }
+ hasDiff
+ }
+ }
+
object InfoBitLimit extends DebugCommand with DebugCommandValidateZeroArgs with InfoSimpleValue[Long] {
val name = "bitLimit"
override lazy val short = "bl"
@@ -1470,14 +1511,20 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
val desc = "display differences since the previous pause in the debugger"
val longDesc = desc
- private lazy val infoDiffables = Info.subcommands.collect { case diffable: InfoDiffable => diffable }
+ lazy val infoDiffables = Info.subcommands.collect { case diffable: InfoDiffable => diffable }
def act(args: Seq[String], state: ParseOrUnparseState, processor: Processor): DebugState.Type = {
debugPrintln("%s:".format(name))
- val differencesExist = infoDiffables.foldLeft(false) { case (foundDiff, ic) =>
- ic.diff(previousProcessorState, state) || foundDiff
+ val foundDiff = infoDiffables.foldLeft(false) { case (prevCmdsFoundDiff, curCmd) =>
+ val curCmdFoundDiff =
+ if (DebuggerConfig.diffExcludes.contains(curCmd.name)) {
+ false // skip current command since it's excluded
+ } else {
+ curCmd.diff(previousProcessorState, state)
+ }
+ curCmdFoundDiff || prevCmdsFoundDiff
}
- if (!differencesExist) {
+ if (!foundDiff) {
debugPrintln("(no differences)", " ")
}
@@ -1666,6 +1713,15 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
}
}
+ object InfoSuspensions extends DebugCommand with DebugCommandValidateZeroArgs with InfoSeqValue[Suspension] {
+ val name = "suspensions"
+ override lazy val short = "sus"
+ val desc = "display list of suspensions"
+ val longDesc = desc
+
+ def getSeqValue(state: StateForDebugger): Seq[Suspension] = state.suspensions
+ }
+
object InfoUnparser extends {
override val name = "unparser"
} with InfoProcessorBase
@@ -1763,7 +1819,17 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
|
|Example: set breakOnlyOnCreation false
| set dataLength 100""".stripMargin
- override val subcommands = Seq(SetBreakOnFailure, SetBreakOnlyOnCreation, SetDataLength, SetInfosetLines, SetInfosetParents, SetRemoveHidden, SetRepresentation, SetWrapLength)
+ override val subcommands = Seq(
+ SetBreakOnFailure,
+ SetBreakOnlyOnCreation,
+ SetDataLength,
+ SetDiffExcludes,
+ SetInfosetLines,
+ SetInfosetParents,
+ SetRemoveHidden,
+ SetRepresentation,
+ SetWrapLength,
+ )
override lazy val short = "set"
def act(args: Seq[String], state: ParseOrUnparseState, processor: Processor): DebugState.Type = {
@@ -1842,6 +1908,32 @@ class InteractiveDebugger(runner: InteractiveDebuggerRunner, eCompilers: Express
}
}
+ object SetDiffExcludes extends DebugCommand {
+ val name = "diffExcludes"
+ val desc = "set info commands to exclude in the 'info diff' commanad"
+ val longDesc = """|Usage: set diffExcludes|de <commands...>
+ |
+ |Set info comamnds to exclude in the 'info diff' command. Multiple arguments
+ |separated by a space excludes multiple commands. Zero arguments excludes no
+ |commands.
+ |
+ |Example: set diffExcludes bitPosition bitLimit""".stripMargin
+ override lazy val short = "de"
+
+ override def validate(args: Seq[String]): Unit = {
+ val diffableNames = Info.InfoDiff.infoDiffables.map { _.name }
+ val unknown = args.diff(diffableNames)
+ if (unknown.size > 0) {
+ throw new DebugException("unknown or undiffable info commands: " + unknown.mkString(", "))
+ }
+ }
+
+ def act(args: Seq[String], state: ParseOrUnparseState, processor: Processor): DebugState.Type = {
+ DebuggerConfig.diffExcludes = args
+ DebugState.Pause
+ }
+ }
+
object SetInfosetLines extends DebugCommand with DebugCommandValidateInt {
val name = "infosetLines"
val desc = "set the maximum number of lines of the infoset to display (default: -1)"
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala
index dce06e7..57fb81b 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/ProcessorStateBases.scala
@@ -80,6 +80,7 @@ trait StateForDebugger {
def variableMap: VariableMap
def delimitedParseResult: Maybe[dfa.ParseResult]
def withinHiddenNest: Boolean
+ def suspensions: Seq[Suspension]
}
case class TupleForDebugger(
@@ -91,7 +92,8 @@ case class TupleForDebugger(
val arrayPos: Long,
val variableMap: VariableMap,
val delimitedParseResult: Maybe[dfa.ParseResult],
- val withinHiddenNest: Boolean)
+ val withinHiddenNest: Boolean,
+ val suspensions: Seq[Suspension])
extends StateForDebugger
trait SetProcessorMixin {
@@ -440,6 +442,7 @@ abstract class ParseOrUnparseState protected (
variableMap.copy(), // deep copy since variableMap is mutable
delimitedParseResult,
withinHiddenNest,
+ suspensions,
)
}
@@ -571,6 +574,7 @@ final class CompileState(tci: DPathCompileInfo, maybeDataProc: Maybe[DataProcess
def groupPos: Long = 0L
def hasInfoset: Boolean = infoset_.isDefined
def delimitedParseResult = Nope
+ def suspensions = Seq.empty
private lazy val infoset_ : Maybe[DIElement] = Nope
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/SuspensionTracker.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/SuspensionTracker.scala
index 7808d29..269c48e 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/SuspensionTracker.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/SuspensionTracker.scala
@@ -30,6 +30,8 @@ class SuspensionTracker(suspensionWaitYoung: Int, suspensionWaitOld: Int)
private val suspensionsYoung = new Queue[Suspension]
private val suspensionsOld = new Queue[Suspension]
+ def suspensions: Seq[Suspension] = suspensionsYoung.toSeq ++ suspensionsOld.toSeq
+
private var count: Int = 0
private var suspensionStatTracked: Int = 0
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
index 45292dc..0a25927 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/PState.scala
@@ -525,6 +525,8 @@ final class PState private (
}
}
}
+
+ def suspensions = Seq.empty
}
object PState {
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
index 9604803..b450514 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/unparsers/UState.scala
@@ -385,6 +385,8 @@ final class UStateForSuspension(
override def getDecoder(cs: BitsCharset): BitsCharsetDecoder = mainUState.getDecoder(cs)
override def getEncoder(cs: BitsCharset): BitsCharsetEncoder = mainUState.getEncoder(cs)
+ override def suspensions = mainUState.suspensions
+
// override def charBufferDataOutputStream = mainUState.charBufferDataOutputStream
override def withUnparserDataInputStream = mainUState.withUnparserDataInputStream
override def withByteArrayOutputStream = mainUState.withByteArrayOutputStream
@@ -632,6 +634,8 @@ final class UStateMain private (
if (isFinal) suspensionTracker.requireFinal()
}
+ def suspensions = suspensionTracker.suspensions
+
final override def pushTRD(trd: TermRuntimeData) =
inputter.pushTRD(trd)