You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2020/07/21 13:45:48 UTC

[incubator-nlpcraft] 01/01: Reading streams approach refactored.

This is an automated email from the ASF dual-hosted git repository.

sergeykamov pushed a commit to branch NLPCRAFT-90
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git

commit 035bf7df47e78d484ff4b214439bbdbab9df4701
Author: Sergey Kamov <se...@apache.org>
AuthorDate: Tue Jul 21 16:45:35 2020 +0300

    Reading streams approach refactored.
---
 .../org/apache/nlpcraft/common/util/NCUtils.scala  | 172 ++++++++-------------
 .../server/geo/tools/NCGeoNamesGenerator.scala     |   2 +-
 .../geo/tools/unstats/NCUnsdStatsService.scala     |   4 +-
 3 files changed, 67 insertions(+), 111 deletions(-)

diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
index 620dfdc..1df5b55 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/common/util/NCUtils.scala
@@ -29,8 +29,8 @@ import java.time.{Instant, ZoneId, ZonedDateTime}
 import java.util.concurrent.{ExecutorService, TimeUnit}
 import java.util.jar.JarFile
 import java.util.stream.Collectors
-import java.util.zip.{ZipInputStream, GZIPInputStream => GIS, GZIPOutputStream ⇒ GOS}
-import java.util.{Locale, Properties, Random, Timer, TimerTask, Calendar ⇒ C}
+import java.util.zip.{ZipInputStream, GZIPInputStream => GIS, GZIPOutputStream => GOS}
+import java.util.{Locale, Properties, Random, Timer, TimerTask, Calendar => C}
 
 import com.fasterxml.jackson.annotation.JsonInclude.Include
 import com.fasterxml.jackson.core.`type`.TypeReference
@@ -44,6 +44,7 @@ import org.apache.commons.codec.digest.DigestUtils
 import org.apache.commons.io.IOUtils
 import org.apache.nlpcraft.common._
 import org.apache.nlpcraft.common.blowfish.NCBlowfishHasher
+import org.apache.nlpcraft.common.util.NCUtils.getStream
 import resource._
 
 import scala.annotation.tailrec
@@ -289,7 +290,7 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+    def readPath(path: String, enc: String, log: Logger = logger): List[String] =
         readFile(new File(path), enc, log)
 
     /**
@@ -300,8 +301,7 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
-        readStream(getStream(res), enc, log)
+    def readResource(res: String, enc: String, log: Logger = logger): List[String] = readStream(getStream(res), enc, log)
 
     /**
       * Reads lines from given file.
@@ -311,7 +311,7 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readGzipPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+    def readGzipPath(path: String, enc: String, log: Logger = logger): List[String] =
         readGzipFile(new File(path), enc, log)
 
     /**
@@ -322,22 +322,14 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readFile(f: File, enc: String, log: Logger = logger): Iterator[String] = {
-        var src: Source = null
-
-        try {
-            src = Source.fromFile(f, enc)
-
-            val data = src.getLines().map(p ⇒ p)
-
-            log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
-            data
-        }
+    def readFile(f: File, enc: String, log: Logger = logger): List[String] =
+        try
+            managed(Source.fromFile(f, enc)) acquireAndGet { src ⇒
+                getAndLog(src.getLines().map(p ⇒ p).toList, f, log)
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
         }
-    }
 
     /**
       * Reads lines from given stream.
@@ -347,9 +339,11 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readStream(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
+    def readStream(in: InputStream, enc: String, log: Logger = logger): List[String] =
         try
-            Source.fromInputStream(in, enc).getLines().map(p ⇒ p)
+            managed(Source.fromInputStream(in, enc)) acquireAndGet { src ⇒
+                src.getLines().map(p ⇒ p).toList
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
         }
@@ -363,22 +357,18 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readTextFile(f: File, enc: String, log: Logger = logger): Iterator[String] = {
-        var src: Source = null
-
-        try {
-            src = Source.fromFile(f, enc)
-
-            val data = src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
-
-            log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
-            data
-        }
+    def readTextFile(f: File, enc: String, log: Logger = logger): List[String] =
+        try
+            managed(Source.fromFile(f, enc)) acquireAndGet { src ⇒
+                getAndLog(
+                    src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList,
+                    f,
+                    logger
+                )
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
         }
-    }
 
     /**
       * Reads lines from given stream converting to lower case, trimming, and filtering
@@ -389,11 +379,11 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readTextStream(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
-        try {
-            Source.fromInputStream(in, enc).getLines().map(_.toLowerCase.trim).
-                filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
-        }
+    def readTextStream(in: InputStream, enc: String, log: Logger = logger): List[String] =
+        try
+            managed(Source.fromInputStream(in, enc)) acquireAndGet { src ⇒
+                src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
         }
@@ -407,15 +397,15 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readTextGzipFile(f: File, enc: String, log: Logger = logger): Iterator[String] =
-        try {
-            val data = Source.fromInputStream(new GIS(new FileInputStream(f)), enc).getLines().map(_.toLowerCase.trim).
-                filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
-
-            log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
-            data
-        }
+    def readTextGzipFile(f: File, enc: String, log: Logger = logger): List[String] =
+        try
+            managed(Source.fromInputStream(new GIS(new FileInputStream(f)), enc)) acquireAndGet { src ⇒
+                getAndLog(
+                    src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList,
+                    f,
+                    log
+                )
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
         }
@@ -429,10 +419,11 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readTextGzipResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
+    def readTextGzipResource(res: String, enc: String, log: Logger = logger): List[String] =
         try
-            Source.fromInputStream(new GIS(getStream(res)), enc).getLines().map(_.toLowerCase.trim).
-                filter(s ⇒ !s.isEmpty && !s.startsWith("#"))
+            managed(Source.fromInputStream(new GIS(getStream(res)), enc)) acquireAndGet { src ⇒
+                src.getLines().map(_.toLowerCase.trim).filter(s ⇒ !s.isEmpty && !s.startsWith("#")).toList
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
         }
@@ -446,7 +437,7 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readTextPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+    def readTextPath(path: String, enc: String, log: Logger = logger): List[String] =
         readTextFile(new File(path), enc, log)
 
     /**
@@ -458,7 +449,7 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readTextResource(res: String, enc: String, log: Logger = logger): Iterator[String] =
+    def readTextResource(res: String, enc: String, log: Logger = logger): List[String] =
         readTextStream(getStream(res), enc, log)
 
     /**
@@ -470,21 +461,10 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readTextGzipPath(path: String, enc: String, log: Logger = logger): Iterator[String] =
+    def readTextGzipPath(path: String, enc: String, log: Logger = logger): List[String] =
         readTextGzipFile(new File(path), enc, log)
 
     /**
-      *
-      * @param path Folder path to read from.
-      * @param enc Encoding.
-      * @param log Logger to use.
-      */
-    @throws[NCE]
-    def readTextFolder(path: String, enc: String, log: Logger = logger): Iterator[String] =
-        (for (file ← new File(path).listFiles()) yield
-            readTextFile(file, enc, logger)).toIterator.flatten
-
-    /**
       * Converts given name into properly capitalized first and last name.
       *
       * @param name Full name.
@@ -566,14 +546,11 @@ object NCUtils extends LazyLogging {
       * @param log Logger to use.
       */
     @throws[NCE]
-    def readGzipFile(f: File, enc: String, log: Logger = logger): Iterator[String] =
-        try {
-            val data = Source.fromInputStream(new GIS(new FileInputStream(f)), enc).getLines().map(p ⇒ p)
-
-            log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
-            data
-        }
+    def readGzipFile(f: File, enc: String, log: Logger = logger): List[String] =
+        try
+            managed(Source.fromInputStream(new GIS(new FileInputStream(f)), enc)) acquireAndGet { src ⇒
+                getAndLog(src.getLines().map(p ⇒ p).toList, f, log)
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
         }
@@ -586,9 +563,11 @@ object NCUtils extends LazyLogging {
       * @return
       */
     @throws[NCE]
-    def readGzipResource(in: InputStream, enc: String, log: Logger = logger): Iterator[String] =
+    def readGzipResource(in: InputStream, enc: String, log: Logger = logger): List[String] =
         try
-            Source.fromInputStream(new GIS(in), enc).getLines().map(p ⇒ p)
+            managed(Source.fromInputStream(new GIS(in), enc)) acquireAndGet { src ⇒
+                src.getLines().map(p ⇒ p).toList
+            }
         catch {
             case e: IOException ⇒ throw new NCE(s"Failed to read stream", e)
         }
@@ -600,8 +579,7 @@ object NCUtils extends LazyLogging {
       * @param log Logger.
       */
     @throws[NCE]
-    def readPathBytes(path: String, log: Logger = logger): Array[Byte] =
-        readFileBytes(new File(path), log)
+    def readPathBytes(path: String, log: Logger = logger): Array[Byte] = readFileBytes(new File(path), log)
 
     /**
       * Reads bytes from given file.
@@ -618,9 +596,7 @@ object NCUtils extends LazyLogging {
                 in.read(arr)
             }
 
-            logger.trace(s"File read: $f")
-
-            arr
+            getAndLog(arr, f, log)
         }
         catch {
             case e: IOException ⇒ throw new NCE(s"Error reading file: $f", e)
@@ -628,40 +604,20 @@ object NCUtils extends LazyLogging {
     }
 
     /**
-      * Reads bytes from given file.
       *
-      * @param f File to read from.
-      * @param log Logger to use.
+      * @param data
+      * @param f
+      * @param log
+      * @tparam T
+      * @return
       */
-    @throws[NCE]
-    def readBinaryFile(f: File, log: Logger = logger): Array[Byte] = {
-        var src: Source = null
+    private def getAndLog[T](data: T, f: File, log: Logger = logger): T = {
+        log.trace(s"Loaded file: ${f.getAbsolutePath}")
 
-        try {
-            src = Source.fromFile(f, "ISO-8859-1")
-
-            val data = src.map(_.toByte).toArray
-
-            log.trace(s"Loaded file: ${f.getAbsolutePath}")
-
-            data
-        }
-        catch {
-            case e: IOException ⇒ throw new NCE(s"Failed to read file: ${f.getAbsolutePath}", e)
-        }
+        data
     }
 
     /**
-      * Reads bytes from given file.
-      *
-      * @param path File path to read from.
-      * @param log Logger to use.
-      */
-    @throws[NCE]
-    def readBinaryPath(path: String, log: Logger = logger): Array[Byte] =
-        readBinaryFile(new File(path), log)
-
-    /**
       * Gzip file.
       *
       * @param f File.
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
index 17c8f09..5e92b50 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/NCGeoNamesGenerator.scala
@@ -74,7 +74,7 @@ object NCGeoNamesGenerator extends App {
     // GEO name ID -> internal representation mapping.
     private val ids = mutable.Map.empty[String, Location]
 
-    private def read(path: String): Iterator[String] = U.readPath(path, "UTF8").filter(!_.startsWith("#"))
+    private def read(path: String): Seq[String] = U.readPath(path, "UTF8").filter(!_.startsWith("#"))
 
     // Process country and continent information.
     private def processCountries(unsdContinents: Seq[NCUnsdStatsContinent]): Set[Country] = {
diff --git a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
index 2a6ebe9..90942d3 100644
--- a/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
+++ b/nlpcraft/src/main/scala/org/apache/nlpcraft/server/geo/tools/unstats/NCUnsdStatsService.scala
@@ -72,7 +72,7 @@ object NCUnsdStatsService {
     private val codesPath = s"$dir/codes.txt"
 
     private def read(path: String): Seq[String] =
-        U.readPath(path, "UTF-8").map(_.trim).filter(_.nonEmpty).filter(_.head != '#').toSeq
+        U.readPath(path, "UTF-8").map(_.trim).filter(_.nonEmpty).filter(_.head != '#')
 
     def skip(iso: String): Boolean = SKIPPED_COUNTRIES_ISO3.contains(iso)
 
@@ -86,7 +86,7 @@ object NCUnsdStatsService {
         }).toMap
 
         val subContsCodes = CONTINENTS.flatMap(_._2).toSeq
-        val contsCodes = CONTINENTS.unzip._1.toSeq
+        val contsCodes = CONTINENTS.keys.toSeq
 
         val buf = mutable.Buffer.empty[NCUnsdStatsContinent]