You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@beam.apache.org by GitBox <gi...@apache.org> on 2022/02/28 08:43:09 UTC
[GitHub] [beam] daria-malkova commented on a change in pull request #16855: [BEAM-13938][Playground] More tests for preparers package
daria-malkova commented on a change in pull request #16855:
URL: https://github.com/apache/beam/pull/16855#discussion_r815666923
##########
File path: playground/backend/internal/preparers/preparer_test.go
##########
@@ -0,0 +1,182 @@
+package preparers
+
+import (
+ pb "beam.apache.org/playground/backend/internal/api/v1"
+ "beam.apache.org/playground/backend/internal/logger"
+ "beam.apache.org/playground/backend/internal/validators"
+ "fmt"
+ "os"
+ "reflect"
+ "sync"
+ "testing"
+)
+
+const (
+ correctGoCode = "package main\n\nimport \"fmt\"\n\nfunc main() {\n\tfmt.Println(\"hello world\")\n\n\n}\n"
+ correctGoFile = "correct.go"
+ incorrectGoCode = "package main\n\nimport \"fmt\"\n\nfunc main() {\n\tfmt.Println(\"hello world\"\n\n}\n"
+ incorrectGoFile = "incorrect.go"
+ pyCode = "import logging as l\n\nif __name__ == \"__main__\":\n logging.info(\"INFO\")\n"
+ correctPyFile = "original.py"
+ incorrectPyFile = "someFile.py"
+ pyGraphCode = "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\"\"\"An example that verifies the counts and includes best practices.\n\nOn top of the basic concepts in the wordcount example, this workflow introduces\nlogging to Cloud
Logging, and using assertions in a Dataflow pipeline.\n\nTo execute this pipeline locally, specify a local output file or output prefix\non GCS::\n\n --output [YOUR_LOCAL_FILE | gs://YOUR_OUTPUT_PREFIX]\n\nTo execute this pipeline using the Google Cloud Dataflow service, specify\npipeline configuration::\n\n --project YOUR_PROJECT_ID\n --staging_location gs://YOUR_STAGING_DIRECTORY\n --temp_location gs://YOUR_TEMP_DIRECTORY\n --region GCE_REGION\n --job_name YOUR_JOB_NAME\n --runner DataflowRunner\n\nand an output prefix on GCS::\n\n --output gs://YOUR_OUTPUT_PREFIX\n\"\"\"\n\n# pytype: skip-file\n\nimport argparse\nimport logging\nimport re\n\nimport apache_beam as beam\nfrom apache_beam.io import ReadFromText\nfrom apache_beam.io import WriteToText\nfrom apache_beam.metrics import Metrics\nfrom apache_beam.options.pipeline_options import PipelineOptions\nfrom apache_beam.options.pipeline_options import SetupOptions\nfrom apache_beam.testing.util import assert_that\nfrom a
pache_beam.testing.util import equal_to\n\n\nclass FilterTextFn(beam.DoFn):\n \"\"\"A DoFn that filters for a specific key based on a regular expression.\"\"\"\n def __init__(self, pattern):\n # TODO(BEAM-6158): Revert the workaround once we can pickle super() on py3.\n # super().__init__()\n beam.DoFn.__init__(self)\n self.pattern = pattern\n # A custom metric can track values in your pipeline as it runs. Those\n # values will be available in the monitoring system of the runner used\n # to run the pipeline. These metrics below track the number of\n # matched and unmatched words.\n self.matched_words = Metrics.counter(self.__class__, 'matched_words')\n self.umatched_words = Metrics.counter(self.__class__, 'umatched_words')\n\n def process(self, element):\n word, _ = element\n if re.match(self.pattern, word):\n # Log at INFO level each element we match. When executing this pipeline\n # using the Dataflow service, these log lines will a
ppear in the Cloud\n # Logging UI.\n logging.info('Matched %s', word)\n self.matched_words.inc()\n yield element\n else:\n # Log at the \"DEBUG\" level each element that is not matched. Different log\n # levels can be used to control the verbosity of logging providing an\n # effective mechanism to filter less important information.\n # Note currently only \"INFO\" and higher level logs are emitted to the\n # Cloud Logger. This log message will not be visible in the Cloud Logger.\n logging.debug('Did not match %s', word)\n self.umatched_words.inc()\n\n\nclass CountWords(beam.PTransform):\n \"\"\"A transform to count the occurrences of each word.\n\n A PTransform that converts a PCollection containing lines of text into a\n PCollection of (word, count) tuples.\n \"\"\"\n def expand(self, pcoll):\n def count_ones(word_ones):\n (word, ones) = word_ones\n return (word, sum(ones))\n\n return (\n pcoll\n
| 'split' >> (\n beam.FlatMap(\n lambda x: re.findall(r'[A-Za-z\\']+', x)).with_output_types(str))\n | 'pair_with_one' >> beam.Map(lambda x: (x, 1))\n | 'group' >> beam.GroupByKey()\n | 'count' >> beam.Map(count_ones))\n\n\ndef run(argv=None, save_main_session=True):\n \"\"\"Runs the debugging wordcount pipeline.\"\"\"\n\n parser = argparse.ArgumentParser()\n parser.add_argument(\n '--input',\n dest='input',\n default='gs://dataflow-samples/shakespeare/kinglear.txt',\n help='Input file to process.')\n parser.add_argument(\n '--output',\n dest='output',\n required=True,\n help='Output file to write results to.')\n known_args, pipeline_args = parser.parse_known_args(argv)\n # We use the save_main_session option because one or more DoFn's in this\n # workflow rely on global context (e.g., a module imported at module level).\n pipeline_options = PipelineOptions(pipeline_args)\n pipelin
e_options.view_as(SetupOptions).save_main_session = save_main_session\n with beam.Pipeline(options=pipeline_options) as p:\n\n # Read the text file[pattern] into a PCollection, count the occurrences of\n # each word and filter by a list of words.\n filtered_words = (\n p | 'read' >> ReadFromText(known_args.input)\n | CountWords()\n | 'FilterText' >> beam.ParDo(FilterTextFn('Flourish|stomach')))\n\n # assert_that is a convenient PTransform that checks a PCollection has an\n # expected value. Asserts are best used in unit tests with small data sets\n # but is demonstrated here as a teaching tool.\n #\n # Note assert_that does not provide any output and that successful\n # completion of the Pipeline implies that the expectations were met. Learn\n # more at https://cloud.google.com/dataflow/pipelines/testing-your-pipeline\n # on how to best test your pipeline.\n assert_that(filtered_words, equal_to([('Flourish', 3), ('stomach',
1)]))\n\n # Format the counts into a PCollection of strings and write the output using\n # a \"Write\" transform that has side effects.\n # pylint: disable=unused-variable\n def format_result(word_count):\n (word, count) = word_count\n return '%s: %s' % (word, count)\n\n output = (\n filtered_words\n | 'format' >> beam.Map(format_result)\n | 'write' >> WriteToText(known_args.output))\n\n\nif __name__ == '__main__':\n # Cloud Logging would contain only logging.INFO and higher level logs logged\n # by the root logger. All log statements emitted by the root logger will be\n # visible in the Cloud Logging UI. Learn more at\n # https://cloud.google.com/logging about the Cloud Logging UI.\n #\n # You can set the default logging level to a different level when running\n # locally.\n logging.getLogger().setLevel(logging.INFO)\n run()\n"
+ pyGraphFile = "graphFile.py"
+)
+
+func TestMain(m *testing.M) {
+ err := setupPreparedFiles()
+ if err != nil {
+ logger.Fatal(err)
+ }
+ defer teardown()
+ m.Run()
+}
+
+// setupPreparedFiles creates 2 go programs:
+// correctFile - program without errors
+// incorrectFile - program with errors
+func setupPreparedFiles() error {
Review comment:
Changed description
##########
File path: playground/backend/internal/preparers/python_preparers_test.go
##########
@@ -101,3 +95,129 @@ func Test_addCodeToFile(t *testing.T) {
})
}
}
+
+func Test_saveLogs(t *testing.T) {
+ file, _ := os.Open(correctPyFile)
+ tmp, _ := utils.CreateTempFile(correctPyFile)
+ defer tmp.Close()
+
+ type args struct {
+ from *os.File
+ to *os.File
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ // Call saveLogs method to add logs code to tmp file.
+ name: "Save logs successfully",
Review comment:
Done
##########
File path: playground/backend/internal/preparers/python_preparers_test.go
##########
@@ -101,3 +95,129 @@ func Test_addCodeToFile(t *testing.T) {
})
}
}
+
+func Test_saveLogs(t *testing.T) {
+ file, _ := os.Open(correctPyFile)
+ tmp, _ := utils.CreateTempFile(correctPyFile)
+ defer tmp.Close()
+
+ type args struct {
+ from *os.File
+ to *os.File
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ // Call saveLogs method to add logs code to tmp file.
+ name: "Save logs successfully",
+ args: args{
+ from: file,
+ to: tmp,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if err := saveLogs(tt.args.from, tt.args.to); (err != nil) != tt.wantErr {
+ t.Errorf("saveLogs() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func Test_writeToFile(t *testing.T) {
+ tmp, _ := utils.CreateTempFile(correctPyFile)
+ defer tmp.Close()
+ tmp2, _ := os.OpenFile(incorrectPyFile, os.O_CREATE, 0)
+ defer tmp2.Close()
+
+ type args struct {
+ to *os.File
+ str string
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ // Call writeToFile method to successfully add line to tmp file.
+ name: "Successfully write to file",
+ args: args{
+ to: tmp,
+ str: "just a string",
+ },
+ wantErr: false,
+ },
+ {
+ // Call writeToFile method to write to tmp file which is open only for reading.
+ // As a result, want to receive error.
+ name: "Write to file which is read-only",
Review comment:
We can't here, cause we can't pass non-existing file to method
##########
File path: playground/backend/internal/preparers/python_preparers_test.go
##########
@@ -101,3 +95,129 @@ func Test_addCodeToFile(t *testing.T) {
})
}
}
+
+func Test_saveLogs(t *testing.T) {
+ file, _ := os.Open(correctPyFile)
+ tmp, _ := utils.CreateTempFile(correctPyFile)
+ defer tmp.Close()
+
+ type args struct {
+ from *os.File
+ to *os.File
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ // Call saveLogs method to add logs code to tmp file.
+ name: "Save logs successfully",
+ args: args{
+ from: file,
+ to: tmp,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if err := saveLogs(tt.args.from, tt.args.to); (err != nil) != tt.wantErr {
+ t.Errorf("saveLogs() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func Test_writeToFile(t *testing.T) {
+ tmp, _ := utils.CreateTempFile(correctPyFile)
+ defer tmp.Close()
+ tmp2, _ := os.OpenFile(incorrectPyFile, os.O_CREATE, 0)
+ defer tmp2.Close()
+
+ type args struct {
+ to *os.File
+ str string
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ }{
+ {
+ // Call writeToFile method to successfully add line to tmp file.
+ name: "Successfully write to file",
+ args: args{
+ to: tmp,
+ str: "just a string",
+ },
+ wantErr: false,
+ },
+ {
+ // Call writeToFile method to write to tmp file which is open only for reading.
+ // As a result, want to receive error.
+ name: "Write to file which is read-only",
+ args: args{
+ to: tmp2,
+ str: "just a string",
+ },
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if err := writeToFile(tt.args.to, tt.args.str); (err != nil) != tt.wantErr {
+ t.Errorf("writeToFile() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
+
+func Test_saveGraph(t *testing.T) {
+ file, _ := os.Open(pyGraphFile)
+ noPipelineFile, _ := os.Open(correctPyFile)
+ tmp, _ := utils.CreateTempFile(pyGraphFile)
+ defer tmp.Close()
+ type args struct {
+ from *os.File
+ tempFile *os.File
+ }
+ tests := []struct {
+ name string
+ args args
+ wantErr bool
+ wantGraph bool
+ }{
+ {
+ // Call saveGraph method to add code (to tmp file) which will save graph to file.
+ name: "Successfully save graph",
+ args: args{
+ from: file,
+ tempFile: tmp,
+ },
+ wantErr: false,
+ wantGraph: true,
+ },
+ {
+ // There is no pipeline definition at the code and no graph code will be added.
Review comment:
We can't, cause we can't pass a non-existing file to the method
##########
File path: playground/backend/internal/preparers/scio_preparers_test.go
##########
@@ -13,39 +13,33 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package utils
+package preparers
import "testing"
-func TestSpacesToEqualsOption(t *testing.T) {
+func TestGetScioPreparers(t *testing.T) {
type args struct {
- pipelineOptions string
+ filePath string
}
tests := []struct {
name string
args args
- want string
+ want int
}{
{
- name: "args is empty string",
- args: args{pipelineOptions: ""},
- want: "",
- },
- {
- name: "args with one option",
- args: args{pipelineOptions: "--opt1 valOpt"},
- want: "--opt1=valOpt",
- },
- {
- name: "args with some options",
- args: args{pipelineOptions: "--opt1 valOpt --opt2 valOpt --opt3 valOpt"},
- want: "--opt1=valOpt --opt2=valOpt --opt3=valOpt",
+ // Test case with calling GetPythonPreparers method.
+ // As a result, want to receive slice of preparers with len = 1
+ name: "get python preparers",
Review comment:
Sure, I fixed the description
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org