You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2023/05/02 13:52:42 UTC
[beam] branch master updated: [Playground] Make fields "categories" and "context_line" optional in examples tags (#26235)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3e5894be029 [Playground] Make fields "categories" and "context_line" optional in examples tags (#26235)
3e5894be029 is described below
commit 3e5894be02945058732a4b8f90a316fee7d7563b
Author: Timur Sultanov <ti...@akvelon.com>
AuthorDate: Tue May 2 17:52:31 2023 +0400
[Playground] Make fields "categories" and "context_line" optional in examples tags (#26235)
* Make context_line and categories optional in the example tag.
* Process multifile examples correctly in GetCatalog()
* Fix example with context line pointing inside of tag
---
.../Core Transforms/Combine/Combine PerKey/task.py | 2 +-
.../db/mapper/precompiled_object_mapper.go | 9 +-
.../db/mapper/precompiled_object_mapper_test.go | 36 ++--
playground/infrastructure/conftest.py | 8 +-
playground/infrastructure/datastore_client.py | 2 +-
playground/infrastructure/helper.py | 6 +-
playground/infrastructure/models.py | 7 +-
playground/infrastructure/test_helper.py | 193 +++++++++++++++++++--
8 files changed, 224 insertions(+), 39 deletions(-)
diff --git a/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py b/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py
index 6ab01d20872..9a112a725a7 100644
--- a/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py
+++ b/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py
@@ -18,7 +18,7 @@
# name: CombinePerKey
# description: Task from katas to implement the summation of scores per player.
# multifile: false
-# context_line: 29
+# context_line: 30
# categories:
# - Combiners
# complexity: BASIC
diff --git a/playground/backend/internal/db/mapper/precompiled_object_mapper.go b/playground/backend/internal/db/mapper/precompiled_object_mapper.go
index cf78a030f92..07481373c3f 100644
--- a/playground/backend/internal/db/mapper/precompiled_object_mapper.go
+++ b/playground/backend/internal/db/mapper/precompiled_object_mapper.go
@@ -61,10 +61,17 @@ func (pom *PrecompiledObjectMapper) ToArrayCategories(catalogDTO *dto.CatalogDTO
numberOfExamples := len(catalogDTO.Examples)
sdkToCategories := make(dto.SdkToCategories, 0)
datasetBySnippetIDMap := catalogDTO.DatasetBySnippetIDMap
+ fileIdx := 0
for exampleIndx := 0; exampleIndx < numberOfExamples; exampleIndx++ {
example := catalogDTO.Examples[exampleIndx]
snippet := catalogDTO.Snippets[exampleIndx]
- files := []*entity.FileEntity{catalogDTO.Files[exampleIndx]}
+
+ var files []*entity.FileEntity
+ for idx := 0; idx < snippet.NumberOfFiles; idx++ {
+ files = append(files, catalogDTO.Files[fileIdx])
+ fileIdx++
+ }
+
var datasetsDTO []*dto.DatasetDTO
if len(datasetBySnippetIDMap) != 0 {
datasetsDTO = datasetBySnippetIDMap[snippet.Key.Name]
diff --git a/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go b/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go
index 7e90d6470c6..40a63714dc0 100644
--- a/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go
+++ b/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go
@@ -40,7 +40,7 @@ func TestPrecompiledObjectMapper_ToObjectInfo(t *testing.T) {
Categories: []string{"MOCK_CAT_1", "MOCK_CAT_2", "MOCK_CAT_3"},
PipelineOptions: "MOCK_OPTIONS",
Link: "MOCK_PATH",
- Multifile: false,
+ Multifile: true,
DefaultExample: false,
ContextLine: 32,
Sdk: pb.Sdk_SDK_JAVA,
@@ -67,7 +67,7 @@ func TestPrecompiledObjectMapper_ToPrecompiledObj(t *testing.T) {
Complexity: pb.Complexity_COMPLEXITY_MEDIUM,
PipelineOptions: "MOCK_OPTIONS",
Link: "MOCK_PATH",
- Multifile: false,
+ Multifile: true,
DefaultExample: false,
ContextLine: 32,
Sdk: pb.Sdk_SDK_JAVA,
@@ -152,7 +152,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) {
javaCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_JAVA_MOCK_NAME" ||
javaCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" ||
javaCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_JAVA_SDK_JAVA_MOCK_NAME" ||
- javaCatalog.Categories[0].PrecompiledObjects[0].Multifile != false ||
+ javaCatalog.Categories[0].PrecompiledObjects[0].Multifile != true ||
javaCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false ||
javaCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" ||
javaCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" ||
@@ -167,7 +167,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) {
goCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_GO_MOCK_NAME" ||
goCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" ||
goCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_GO_SDK_GO_MOCK_NAME" ||
- goCatalog.Categories[0].PrecompiledObjects[0].Multifile != false ||
+ goCatalog.Categories[0].PrecompiledObjects[0].Multifile != true ||
goCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false ||
goCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" ||
goCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" ||
@@ -182,7 +182,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) {
pythonCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_PYTHON_MOCK_NAME" ||
pythonCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" ||
pythonCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_PYTHON_SDK_PYTHON_MOCK_NAME" ||
- pythonCatalog.Categories[0].PrecompiledObjects[0].Multifile != false ||
+ pythonCatalog.Categories[0].PrecompiledObjects[0].Multifile != true ||
pythonCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false ||
pythonCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" ||
pythonCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" ||
@@ -197,7 +197,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) {
scioCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_SCIO_MOCK_NAME" ||
scioCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" ||
scioCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_SCIO_SDK_SCIO_MOCK_NAME" ||
- scioCatalog.Categories[0].PrecompiledObjects[0].Multifile != false ||
+ scioCatalog.Categories[0].PrecompiledObjects[0].Multifile != true ||
scioCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false ||
scioCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" ||
scioCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" ||
@@ -234,15 +234,23 @@ func getExampleDTO(name, defaultName, sdk string) *dto.ExampleDTO {
PipeOpts: "MOCK_OPTIONS",
Origin: constants.ExampleOrigin,
SchVer: utils.GetSchemaVerKey(pcObjMapperCtx, "MOCK_VERSION"),
- NumberOfFiles: 1,
+ NumberOfFiles: 2,
Complexity: pb.Complexity_COMPLEXITY_MEDIUM.String(),
},
- Files: []*entity.FileEntity{{
- Name: "MOCK_NAME",
- Content: "MOCK_CONTENT",
- CntxLine: 32,
- IsMain: true,
- }},
+ Files: []*entity.FileEntity{
+ {
+ Name: "MOCK_NAME_0",
+ Content: "MOCK_CONTENT",
+ CntxLine: 32,
+ IsMain: true,
+ },
+ {
+ Name: "MOCK_NAME_1",
+ Content: "MOCK_CONTENT",
+ CntxLine: 16,
+ IsMain: false,
+ },
+ },
DefaultExampleName: defaultName,
Datasets: []*dto.DatasetDTO{
{
@@ -288,7 +296,7 @@ func getCatalogDTO() *dto.CatalogDTO {
exampleDTO := getExampleDTO(utils.GetIDWithDelimiter(sdk.Name, "MOCK_NAME"), "MOCK_DEFAULT_EXAMPLE", sdk.Name)
examples = append(examples, exampleDTO.Example)
snippets = append(snippets, exampleDTO.Snippet)
- files = append(files, exampleDTO.Files[0])
+ files = append(files, exampleDTO.Files...)
}
return &dto.CatalogDTO{
Examples: examples,
diff --git a/playground/infrastructure/conftest.py b/playground/infrastructure/conftest.py
index face74f0e2b..73db43fe085 100644
--- a/playground/infrastructure/conftest.py
+++ b/playground/infrastructure/conftest.py
@@ -78,7 +78,7 @@ def create_test_example(create_test_tag):
@pytest.fixture
def create_test_tag():
- def _create_test_tag(with_kafka=False, is_multifile=False, **tag_meta) -> Tag:
+ def _create_test_tag(with_kafka=False, is_multifile=False, context_line=30, line_start=10, line_finish=20, **tag_meta) -> Tag:
meta = {
"name": "MOCK_NAME",
"description": "MOCK_DESCRIPTION",
@@ -112,9 +112,9 @@ def create_test_tag():
meta[k] = v
return Tag(
filepath="../../examples/MOCK_EXAMPLE/main.java",
- line_start=10,
- line_finish=20,
- context_line=30,
+ line_start=line_start,
+ line_finish=line_finish,
+ context_line=context_line,
**meta,
)
diff --git a/playground/infrastructure/datastore_client.py b/playground/infrastructure/datastore_client.py
index d5f59ce28bf..8cb03ba709a 100644
--- a/playground/infrastructure/datastore_client.py
+++ b/playground/infrastructure/datastore_client.py
@@ -352,7 +352,7 @@ class DatastoreClient:
example.tag.name, example.sdk
),
"content": example.code,
- "cntxLine": example.tag.context_line,
+ "cntxLine": example.context_line,
"isMain": True,
}
)
diff --git a/playground/infrastructure/helper.py b/playground/infrastructure/helper.py
index 50ed4a20ea4..581d9d5c201 100644
--- a/playground/infrastructure/helper.py
+++ b/playground/infrastructure/helper.py
@@ -283,6 +283,10 @@ def _get_example(filepath: str, filename: str, tag: Tag, sdk: int) -> Example:
Returns:
Parsed Example object.
"""
+
+ # Calculate context line with tag removed. Note: context_line is 1-based, line_start and line_finish are 0-based.
+ context_line = tag.context_line if tag.context_line <= tag.line_start else tag.context_line - (tag.line_finish - tag.line_start)
+
return Example(
sdk=SdkEnum(sdk),
tag=tag,
@@ -291,7 +295,7 @@ def _get_example(filepath: str, filename: str, tag: Tag, sdk: int) -> Example:
type=_get_object_type(filename, filepath),
code=_get_content(filepath, tag.line_start, tag.line_finish),
url_vcs=_get_url_vcs(filepath), # type: ignore
- context_line=tag.context_line - (tag.line_finish - tag.line_start),
+ context_line=context_line,
)
diff --git a/playground/infrastructure/models.py b/playground/infrastructure/models.py
index 904a3e14058..bf667eeac38 100644
--- a/playground/infrastructure/models.py
+++ b/playground/infrastructure/models.py
@@ -75,11 +75,11 @@ class Tag(BaseModel):
filepath: str = Field(..., min_length=1)
line_start: int
line_finish: int
- context_line: int
+ context_line: int = 1
name: str = Field(..., min_length=1)
complexity: ComplexityEnum
description: str
- categories: List[str]
+ categories: List[str] = []
pipeline_options: str = ""
datasets: Dict[str, Dataset] = {}
emulators: List[Emulator] = []
@@ -98,7 +98,8 @@ class Tag(BaseModel):
@root_validator(skip_on_failure=True)
def lines_order(cls, values):
assert (
- 0 < values["line_start"] < values["line_finish"] <= values["context_line"]
+ (0 <= values["line_start"] < values["line_finish"]) and
+ (values["context_line"] <= values["line_start"] or values["context_line"] > values["line_finish"])
), f"line ordering error: {values}"
return values
diff --git a/playground/infrastructure/test_helper.py b/playground/infrastructure/test_helper.py
index 164adf588e1..063f3285a61 100644
--- a/playground/infrastructure/test_helper.py
+++ b/playground/infrastructure/test_helper.py
@@ -129,8 +129,7 @@ async def test_get_statuses(mock_update_example_status, create_test_example):
@mock.patch(
"builtins.open",
mock_open(
- read_data="""
-// license line 1
+ read_data="""// license line 1
// license line 2
//
// beam-playground:
@@ -171,8 +170,7 @@ def test_load_example():
sdk=SdkEnum.JAVA,
type=PRECOMPILED_OBJECT_TYPE_EXAMPLE,
filepath="../../examples/MOCK_EXAMPLE/main.java",
- code="""
-// license line 1
+ code="""// license line 1
// license line 2
//
@@ -184,8 +182,8 @@ code line 2
context_line=5,
tag=Tag(
filepath="../../examples/MOCK_EXAMPLE/main.java",
- line_start=4,
- line_finish=27,
+ line_start=3,
+ line_finish=26,
name="KafkaWordCount",
description="Test example with Apache Kafka",
multifile=False,
@@ -208,6 +206,181 @@ code line 2
)
+@mock.patch(
+ "builtins.open",
+ mock_open(
+ read_data="""// license line 1
+// license line 2
+//
+// beam-playground:
+// name: KafkaWordCount
+// description: Test example with Apache Kafka
+// multifile: false
+// context_line: 27
+// categories:
+// - Filtering
+// - Options
+// - Quickstart
+// complexity: MEDIUM
+// tags:
+// - filter
+// - strings
+// - emulator
+// emulators:
+// - type: kafka
+// topic:
+// id: topic_1
+// source_dataset: dataset_id_1
+// datasets:
+// dataset_id_1:
+// location: local
+// format: json
+
+code line 1
+code line 2
+
+"""
+ ),
+)
+def test_load_example_context_at_the_end_of_tag():
+ example = _load_example(
+ "kafka.java", "../../examples/MOCK_EXAMPLE/main.java", SdkEnum.JAVA
+ )
+ assert example == Example(
+ sdk=SdkEnum.JAVA,
+ type=PRECOMPILED_OBJECT_TYPE_EXAMPLE,
+ filepath="../../examples/MOCK_EXAMPLE/main.java",
+ code="""// license line 1
+// license line 2
+//
+
+code line 1
+code line 2
+
+""",
+ url_vcs="https://github.com/apache/beam/blob/master/examples/MOCK_EXAMPLE/main.java", # type: ignore
+ context_line=4,
+ tag=Tag(
+ filepath="../../examples/MOCK_EXAMPLE/main.java",
+ line_start=3,
+ line_finish=26,
+ name="KafkaWordCount",
+ description="Test example with Apache Kafka",
+ multifile=False,
+ context_line=27,
+ categories=["Filtering", "Options", "Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["filter", "strings", "emulator"],
+ emulators=[
+ Emulator(
+ type=EmulatorType.KAFKA,
+ topic=Topic(id="topic_1", source_dataset="dataset_id_1"),
+ )
+ ],
+ datasets={
+ "dataset_id_1": Dataset(
+ location=DatasetLocation.LOCAL, format=DatasetFormat.JSON
+ )
+ },
+ ),
+ )
+
+@mock.patch(
+ "builtins.open",
+ mock_open(
+ read_data="""// license line 1
+// license line 2
+//
+// beam-playground:
+// name: KafkaWordCount
+// description: Test example with Apache Kafka
+// multifile: false
+// context_line: 3
+// categories:
+// - Filtering
+// - Options
+// - Quickstart
+// complexity: MEDIUM
+// tags:
+// - filter
+// - strings
+// - emulator
+// emulators:
+// - type: kafka
+// topic:
+// id: topic_1
+// source_dataset: dataset_id_1
+// datasets:
+// dataset_id_1:
+// location: local
+// format: json
+
+code line 1
+code line 2
+
+"""
+ ),
+)
+def test_load_example_context_before_of_tag():
+ example = _load_example(
+ "kafka.java", "../../examples/MOCK_EXAMPLE/main.java", SdkEnum.JAVA
+ )
+ assert example == Example(
+ sdk=SdkEnum.JAVA,
+ type=PRECOMPILED_OBJECT_TYPE_EXAMPLE,
+ filepath="../../examples/MOCK_EXAMPLE/main.java",
+ code="""// license line 1
+// license line 2
+//
+
+code line 1
+code line 2
+
+""",
+ url_vcs="https://github.com/apache/beam/blob/master/examples/MOCK_EXAMPLE/main.java", # type: ignore
+ context_line=3,
+ tag=Tag(
+ filepath="../../examples/MOCK_EXAMPLE/main.java",
+ line_start=3,
+ line_finish=26,
+ name="KafkaWordCount",
+ description="Test example with Apache Kafka",
+ multifile=False,
+ context_line=3,
+ categories=["Filtering", "Options", "Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["filter", "strings", "emulator"],
+ emulators=[
+ Emulator(
+ type=EmulatorType.KAFKA,
+ topic=Topic(id="topic_1", source_dataset="dataset_id_1"),
+ )
+ ],
+ datasets={
+ "dataset_id_1": Dataset(
+ location=DatasetLocation.LOCAL, format=DatasetFormat.JSON
+ )
+ },
+ ),
+ )
+
+
+def test__validate_context_line_at_beggining_of_tag(create_test_tag):
+ with pytest.raises(
+ pydantic.ValidationError,
+ match="line ordering error",
+ ):
+ create_test_tag(context_line=4, line_start=3, line_finish=27)
+
+
+def test__validate_context_line_at_end_of_tag(create_test_tag):
+ with pytest.raises(
+ pydantic.ValidationError,
+ match="line ordering error",
+ ):
+ create_test_tag(context_line=27, line_start=4, line_finish=27)
+
+
def test__validate_without_name_field(create_test_tag):
with pytest.raises(
pydantic.ValidationError,
@@ -232,14 +405,6 @@ def test__validate_with_incorrect_multifile_field(create_test_tag):
create_test_tag(multifile="multifile")
-def test__validate_without_categories_field(create_test_tag):
- with pytest.raises(
- pydantic.ValidationError,
- match="field required",
- ):
- create_test_tag(categories=None)
-
-
def test__validate_with_incorrect_categories_field(create_test_tag):
with pytest.raises(
pydantic.ValidationError,