You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@daffodil.apache.org by GitBox <gi...@apache.org> on 2021/06/28 20:35:19 UTC

[GitHub] [daffodil] mbeckerle commented on a change in pull request #599: Move TDML processor cache from a global object to DFDLTestSuite instances

mbeckerle commented on a change in pull request #599:
URL: https://github.com/apache/daffodil/pull/599#discussion_r660077349



##########
File path: daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala
##########
@@ -91,75 +98,127 @@ object Runner {
  * test can be run in parallel.
  *
  * Note however, that each thread will get its own copy of the DFDLTestSuite

Review comment:
       Is this Note above still true? They all share one which synchronizes the schema compile method don't they?

##########
File path: daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
##########
@@ -119,14 +119,24 @@ trait TDMLDFDLProcessor {
 
   def unparse(parseResult: TDMLParseResult, outStream: java.io.OutputStream): TDMLUnparseResult
 
+  /**
+   * The TDMLRunner may cache the TDMLDFDLProcessor so that it can be reused
+   * among multiple parse/unparse calls without needing to rebuild the
+   * processor. This function is called once the TDMLRunner is done with the
+   * TMLDFDLProcessor. This allows state such as temporary files to be cleaned
+   * up. Actually does nothing unless overridden by an implementation.
+   */
+  def cleanUp(): Unit = { /* Do nothing */ }

Review comment:
       Should be protected method? 

##########
File path: daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala
##########
@@ -91,75 +98,127 @@ object Runner {
  * test can be run in parallel.
  *
  * Note however, that each thread will get its own copy of the DFDLTestSuite
+ *
+ * source is either a scala.xml.Elem, generally used for unit testing or a
+ * String. If the string starts with a / is it treated is if it were resoucrce
+ * on the classpath. Otherwise it is treated as if it were a URI.
  */
-class Runner private (elem: scala.xml.Elem, dir: String, file: String,
+class Runner private (
+  source: Either[scala.xml.Elem, String],

Review comment:
       You said you got rid of support for using an Elem here. Do we still need this Either then?
   Ah. I see you still allow the whole test suite to be an Elem, just not a schema. 
   

##########
File path: daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala
##########
@@ -91,75 +98,127 @@ object Runner {
  * test can be run in parallel.
  *
  * Note however, that each thread will get its own copy of the DFDLTestSuite
+ *
+ * source is either a scala.xml.Elem, generally used for unit testing or a
+ * String. If the string starts with a / is it treated is if it were resoucrce
+ * on the classpath. Otherwise it is treated as if it were a URI.
  */
-class Runner private (elem: scala.xml.Elem, dir: String, file: String,
+class Runner private (
+  source: Either[scala.xml.Elem, String],
   validateTDMLFile: Boolean = true,
   validateDFDLSchemas: Boolean = true,
   compileAllTopLevel: Boolean = false,
   defaultRoundTripDefault: RoundTrip = Runner.defaultRoundTripDefaultDefault,
   defaultValidationDefault: String = Runner.defaultValidationDefaultDefault,
   defaultImplementationsDefault: Seq[String] = Runner.defaultImplementationsDefaultDefault) {
 
-  /*
-   * these constructors are for use by Java programs
+
+  /**
+   * Create a runner for the path. This constructor requires this path be on
+   * the classpath. A forward slash is prepended to ensure this path aways
+   * works as a classpath resource.
    */
-  def this(dir: String, file:String) =
-    this(null, dir, file)
+  def this(path: String) =
+    this(
+      if (path.startsWith("/")) Right(path)
+      else Right("/" + path)

Review comment:
       Hmmm. Not sure I would add tests showing this auto pre-slashifies the path. If this isn't covered I think I'd rather change this to assert that the path must startwith "/" and fail otherwise. 
   
   OTOH. I guess others may have TDML files that depend on this behavior. So I guess a test would be good. You can just modify some other test this way, and have that test also exercise this little feature. 

##########
File path: daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestOVCAndLength.scala
##########
@@ -19,14 +19,10 @@ package org.apache.daffodil.unparser
 
 import org.junit.Test
 import org.junit.AfterClass
-import org.apache.daffodil.util._
-import org.apache.daffodil.tdml.DFDLTestSuite
+import org.apache.daffodil.tdml.Runner
 
 object TestOVCAndLength {
-  val testDir = "/org/apache/daffodil/unparser/"
-  val aa = testDir + "OVCAndLengthTest.tdml"
-  var runner = new DFDLTestSuite(Misc.getRequiredResource(aa))
-
+  var runner = Runner("/org/apache/daffodil/unparser/OVCAndLengthTest.tdml")

Review comment:
       should be val

##########
File path: daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/RunnerFactory.scala
##########
@@ -91,75 +98,127 @@ object Runner {
  * test can be run in parallel.
  *
  * Note however, that each thread will get its own copy of the DFDLTestSuite
+ *
+ * source is either a scala.xml.Elem, generally used for unit testing or a
+ * String. If the string starts with a / is it treated is if it were resoucrce
+ * on the classpath. Otherwise it is treated as if it were a URI.
  */
-class Runner private (elem: scala.xml.Elem, dir: String, file: String,
+class Runner private (
+  source: Either[scala.xml.Elem, String],
   validateTDMLFile: Boolean = true,
   validateDFDLSchemas: Boolean = true,
   compileAllTopLevel: Boolean = false,
   defaultRoundTripDefault: RoundTrip = Runner.defaultRoundTripDefaultDefault,
   defaultValidationDefault: String = Runner.defaultValidationDefaultDefault,
   defaultImplementationsDefault: Seq[String] = Runner.defaultImplementationsDefaultDefault) {
 
-  /*
-   * these constructors are for use by Java programs
+
+  /**
+   * Create a runner for the path. This constructor requires this path be on
+   * the classpath. A forward slash is prepended to ensure this path aways
+   * works as a classpath resource.
    */
-  def this(dir: String, file:String) =
-    this(null, dir, file)
+  def this(path: String) =
+    this(
+      if (path.startsWith("/")) Right(path)
+      else Right("/" + path)
+    )
 
+  /**
+   * Create a runner for the dir + file combination. This constructor requires
+   * this combined path be on the classpath. A forward slash is prepended to

Review comment:
       This comment is wrong. It's a cut paste from prior method doc. This forward slash is appended if needed, but it doesn't have to do with classpath. Just has to do with making sure the dir and filename are separated by a "/". 

##########
File path: daffodil-tdml-lib/src/main/scala/org/apache/daffodil/tdml/processor/TDMLDFDLProcessor.scala
##########
@@ -119,14 +119,24 @@ trait TDMLDFDLProcessor {
 
   def unparse(parseResult: TDMLParseResult, outStream: java.io.OutputStream): TDMLUnparseResult
 
+  /**
+   * The TDMLRunner may cache the TDMLDFDLProcessor so that it can be reused
+   * among multiple parse/unparse calls without needing to rebuild the
+   * processor. This function is called once the TDMLRunner is done with the
+   * TMLDFDLProcessor. This allows state such as temporary files to be cleaned
+   * up. Actually does nothing unless overridden by an implementation.
+   */
+  def cleanUp(): Unit = { /* Do nothing */ }
 }
 
 trait TDMLResult {
   def isValidationError: Boolean
   def isProcessingError: Boolean
   def getDiagnostics: Seq[Diagnostic]
+
   /**
-   * Deletes any temporary files that have been generated.
+   * Called once the TDMLRunner has finished analyzing the parse/unparse
+   * result. This allows state such as temporary files to be cleaned up.
    * Actually does nothing unless overridden by an implementation.
    */
   def cleanUp(): Unit = { /* Do nothing */ }

Review comment:
       protected?

##########
File path: daffodil-tdml-processor/src/main/scala/org/apache/daffodil/tdml/processor/Runtime2TDMLDFDLProcessor.scala
##########
@@ -190,20 +191,24 @@ class Runtime2TDMLDFDLProcessor(tempDir: os.Path, executable: os.Path,
   // the unparsed data on its standard output, and write any error messages
   // on its standard output (all done in [[Runtime2DataProcessor.unparse]]).
   override def unparse(infosetXML: scala.xml.Node, outStream: java.io.OutputStream): TDMLUnparseResult = {
-    val tempInputFile = XMLUtils.convertNodeToTempFile(infosetXML, tempDir.toIO)
-    val inStream = os.read.inputStream(os.Path(tempInputFile))
+    val inStream = new ByteArrayInputStream(infosetXML.toString.getBytes())
     val upr = dataProcessor.unparse(inStream, outStream)
     anyErrors = upr.isError
     diagnostics = diagnostics ++ upr.getDiagnostics
-    new Runtime2TDMLUnparseResult(upr, tempDir)
+    new Runtime2TDMLUnparseResult(upr)
   }
 
   def unparse(parseResult: TDMLParseResult, outStream: java.io.OutputStream): TDMLUnparseResult = {
     unparse(parseResult.getResult, outStream)
   }
+
+  /**
+   * Remove the generated executable
+   */
+  override def cleanUp(): Unit = os.remove.all(tempDir)

Review comment:
       override protected def ?

##########
File path: daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunner.scala
##########
@@ -68,34 +69,30 @@ class TestTDMLRunner {
                       </parserTestCase>
                     </testSuite>
 
-    using(new java.io.FileWriter(tmp)) {
-      fileWriter =>
-        fileWriter.write(testSchema.toString())
-    }
-
-    // compileAllTopLevel must be true to enable caching
-    lazy val ts = new DFDLTestSuite(testSuite, true, true, true)
+    lazy val runner = new Runner(testSuite)
 
     // Run the non-validation test first, this will compile, serialize, and
     // cache the schema
-    ts.runOneTest("test3a")
+    runner.runOneTest("test3a")
 
     // Run the validation test second. This should recompile and cache the
     // schema separately from from the first test. If it ended up using the
     // non-validation cached Processor, an exception would be thrown since it
     // was serialized and that does not work with validation="on"
-    ts.runOneTest("test3b")
-  }
+    runner.runOneTest("test3b")
 
-  val testSchema = SchemaUtils.dfdlTestSchema(
-    <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>,
-    <dfdl:format ref="tns:GeneralFormat"/>,
-    <xs:element name="data" type="xs:int" dfdl:lengthKind="explicit" dfdl:length="{ xs:unsignedInt(2) }"/>)
+    runner.reset
+  }
 
   @Test def testTDMLParseSuccess(): Unit = {
 
-    val testSuite = <ts:testSuite xmlns:ts={ tdml } suiteName="theSuiteName">
-                      <ts:parserTestCase ID="some identifier" name="testTDMLParseSuccess" root="data">
+    val testSuite = <ts:testSuite xmlns:ts={ tdml } suiteName="theSuiteName" xmlns:xs={ xsd } xmlns:dfdl={ dfdl } xmlns:tns={ example }>
+                      <ts:defineSchema name="mySchema">

Review comment:
       This is a nice cleanup, as all these tests pre-dated our introduction of the ts:defineSchema element to tdml.

##########
File path: daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunnerCommentSyntax.scala
##########
@@ -87,8 +87,9 @@ class TestTDMLRunnerCommentSyntax {
         <!-- comment -->
       </tdml:testSuite>
 
-    lazy val ts = new DFDLTestSuite(testSuite)
-    ts.runOneTest("test1")
+    lazy val runner = new Runner(testSuite)

Review comment:
       not lazy

##########
File path: daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunner.scala
##########
@@ -509,16 +566,16 @@ f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
         </tdml:parserTestCase>
       </tdml:testSuite>
 
-    lazy val ts = new DFDLTestSuite(testSuite)
-    val bytes = ts.parserTestCases(0).document.get.documentBytes
+    lazy val runner = new Runner(testSuite)
+    val bytes = runner.getTS.parserTestCases(0).document.get.documentBytes
     assertEquals(9, bytes.length)
-    ts.runOneTest("testNilCompare")
+    runner.runOneTest("testNilCompare")
   }
 
   @Test def test_tdmlNamespaces1(): Unit = {
     val testDir = "/test/tdml/"
     val t0 = testDir + "tdmlNamespaces.tdml"
-    lazy val r = new DFDLTestSuite(Misc.getRequiredResource(t0))
+    lazy val r = new Runner(t0)

Review comment:
       Not lazy val

##########
File path: daffodil-test/src/test/scala/org/apache/daffodil/unparser/TestOVCAndLength.scala
##########
@@ -19,14 +19,10 @@ package org.apache.daffodil.unparser
 
 import org.junit.Test
 import org.junit.AfterClass
-import org.apache.daffodil.util._
-import org.apache.daffodil.tdml.DFDLTestSuite
+import org.apache.daffodil.tdml.Runner
 
 object TestOVCAndLength {
-  val testDir = "/org/apache/daffodil/unparser/"
-  val aa = testDir + "OVCAndLengthTest.tdml"
-  var runner = new DFDLTestSuite(Misc.getRequiredResource(aa))
-
+  var runner = Runner("/org/apache/daffodil/unparser/OVCAndLengthTest.tdml")
   @AfterClass def tearDown(): Unit = {
     runner = null

Review comment:
       should be runner.reset

##########
File path: daffodil-tdml-processor/src/test/scala/org/apache/daffodil/tdml/TestTDMLRunner.scala
##########
@@ -355,8 +408,9 @@ class TestTDMLRunner {
 
   @Test def testMultiByteUnicodeWorks(): Unit = {
     val testSuite = tdmlWithUnicode2028
-    lazy val ts = new DFDLTestSuite(testSuite)
-    ts.runOneTest("testMultiByteUnicodeWorks")
+    lazy val runner = new Runner(testSuite)

Review comment:
       These don't need to be lazy val. You can probably global replace ```lazy val runner =``` with ```val runner =```




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@daffodil.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org