You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@beam.apache.org by GitBox <gi...@apache.org> on 2021/12/20 10:31:41 UTC
[GitHub] [beam] AydarZaynutdinov opened a new pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
AydarZaynutdinov opened a new pull request #16277:
URL: https://github.com/apache/beam/pull/16277
[BEAM-13124]
Implement handle func for `/readiness` address;
Add deleting all prepared files/folders for `RunCode` Api method in case of any error with the cache;
------------------------
Thank you for your contribution! Follow this checklist to help us incorporate your contribution quickly and easily:
- [ ] [**Choose reviewer(s)**](https://beam.apache.org/contribute/#make-your-change) and mention them in a comment (`R: @username`).
- [ ] Format the pull request title like `[BEAM-XXX] Fixes bug in ApproximateQuantiles`, where you replace `BEAM-XXX` with the appropriate JIRA issue, if applicable. This will automatically link the pull request to the issue.
- [ ] Update `CHANGES.md` with noteworthy changes.
- [ ] If this contribution is large, please file an Apache [Individual Contributor License Agreement](https://www.apache.org/licenses/icla.pdf).
See the [Contributor Guide](https://beam.apache.org/contribute) for more tips on [how to make review process smoother](https://beam.apache.org/contribute/#make-reviewers-job-easier).
`ValidatesRunner` compliance status (on master branch)
--------------------------------------------------------
<table>
<thead>
<tr>
<th>Lang</th>
<th>ULR</th>
<th>Dataflow</th>
<th>Flink</th>
<th>Samza</th>
<th>Spark</th>
<th>Twister2</th>
</tr>
</thead>
<tbody>
<tr>
<td>Go</td>
<td>---</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Go/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Go/lastCompletedBuild/badge/icon">
</a>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Flink/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Samza/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Samza/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Go_VR_Spark/lastCompletedBuild/badge/icon">
</a>
</td>
<td>---</td>
</tr>
<tr>
<td>Java</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_ULR/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_ULR/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow/lastCompletedBuild/badge/icon?subject=V1">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming/lastCompletedBuild/badge/icon?subject=V1+Streaming">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java11/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_Java11/lastCompletedBuild/badge/icon?subject=V1+Java+11">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_VR_Dataflow_V2/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_VR_Dataflow_V2/lastCompletedBuild/badge/icon?subject=V2">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_VR_Dataflow_V2_Streaming/lastCompletedBuild/badge/icon?subject=V2+Streaming">
</a><br>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink/lastCompletedBuild/badge/icon?subject=Java+8">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink_Java11/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink_Java11/lastCompletedBuild/badge/icon?subject=Java+11">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Batch/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Batch/lastCompletedBuild/badge/icon?subject=Portable">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Streaming/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Flink_Streaming/lastCompletedBuild/badge/icon?subject=Portable+Streaming">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Samza/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Samza/lastCompletedBuild/badge/icon">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Samza/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Samza/lastCompletedBuild/badge/icon?subject=Portable">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Spark/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Spark/lastCompletedBuild/badge/icon">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch/lastCompletedBuild/badge/icon?subject=Portable">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming/lastCompletedBuild/badge/icon?subject=Structured+Streaming">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Twister2/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Twister2/lastCompletedBuild/badge/icon">
</a>
</td>
</tr>
<tr>
<td>Python</td>
<td>---</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow/lastCompletedBuild/badge/icon?subject=V1">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Py_VR_Dataflow_V2/lastCompletedBuild/badge/icon?subject=V2">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/lastCompletedBuild/badge/icon?subject=ValCont">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Python_PVR_Flink_Cron/lastCompletedBuild/badge/icon?subject=Portable">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Flink/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Samza/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/lastCompletedBuild/badge/icon">
</a>
</td>
<td>---</td>
</tr>
<tr>
<td>XLang</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Direct/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Direct/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Dataflow/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Dataflow/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Flink/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Flink/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Samza/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Samza/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Spark/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_XVR_Spark/lastCompletedBuild/badge/icon">
</a>
</td>
<td>---</td>
</tr>
</tbody>
</table>
Examples testing status on various runners
--------------------------------------------------------
<table>
<thead>
<tr>
<th>Lang</th>
<th>ULR</th>
<th>Dataflow</th>
<th>Flink</th>
<th>Samza</th>
<th>Spark</th>
<th>Twister2</th>
</tr>
</thead>
<tbody>
<tr>
<td>Go</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
</tr>
<tr>
<td>Java</td>
<td>---</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Cron/lastCompletedBuild/badge/icon?subject=V1">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Java11_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Java_Examples_Dataflow_Java11_Cron/lastCompletedBuild/badge/icon?subject=V1+Java11">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_V2/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_V2/lastCompletedBuild/badge/icon?subject=V2">
</a><br>
</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
</tr>
<tr>
<td>Python</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
</tr>
<tr>
<td>XLang</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
<td>---</td>
</tr>
</tbody>
</table>
Post-Commit SDK/Transform Integration Tests Status (on master branch)
------------------------------------------------------------------------------------------------
<table>
<thead>
<tr>
<th>Go</th>
<th>Java</th>
<th>Python</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Go/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Go/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Java/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Java/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Python36/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Python36/lastCompletedBuild/badge/icon?subject=3.6">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Python37/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Python37/lastCompletedBuild/badge/icon?subject=3.7">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PostCommit_Python38/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PostCommit_Python38/lastCompletedBuild/badge/icon?subject=3.8">
</a>
</td>
</tr>
</tbody>
</table>
Pre-Commit Tests Status (on master branch)
------------------------------------------------------------------------------------------------
<table>
<thead>
<tr>
<th>---</th>
<th>Java</th>
<th>Python</th>
<th>Go</th>
<th>Website</th>
<th>Whitespace</th>
<th>Typescript</th>
</tr>
</thead>
<tbody>
<tr>
<td>Non-portable</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Java_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Java_Cron/lastCompletedBuild/badge/icon">
</a><br>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/lastCompletedBuild/badge/icon?subject=Tests">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron/lastCompletedBuild/badge/icon?subject=Lint">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_PythonDocker_Cron/badge/icon?subject=Docker">
</a><br>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_PythonDocs_Cron/badge/icon?subject=Docs">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Go_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Go_Cron/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Website_Cron/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Whitespace_Cron/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Typescript_Cron/lastCompletedBuild/badge/icon">
</a>
</td>
</tr>
<tr>
<td>Portable</td>
<td>---</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron/lastCompletedBuild/badge/icon">
</a>
</td>
<td>
<a href="https://ci-beam.apache.org/job/beam_PreCommit_GoPortable_Cron/lastCompletedBuild/">
<img alt="Build Status" src="https://ci-beam.apache.org/job/beam_PreCommit_GoPortable_Cron/lastCompletedBuild/badge/icon">
</a>
</td>
<td>---</td>
<td>---</td>
<td>---</td>
</tr>
</tbody>
</table>
See [.test-infra/jenkins/README](https://github.com/apache/beam/blob/master/.test-infra/jenkins/README.md) for trigger phrase, status and link of all Jenkins jobs.
GitHub Actions Tests Status (on master branch)
------------------------------------------------------------------------------------------------
[](https://github.com/apache/beam/actions?query=workflow%3A%22Build+python+source+distribution+and+wheels%22+branch%3Amaster+event%3Aschedule)
[](https://github.com/apache/beam/actions?query=workflow%3A%22Python+Tests%22+branch%3Amaster+event%3Aschedule)
[](https://github.com/apache/beam/actions?query=workflow%3A%22Java+Tests%22+branch%3Amaster+event%3Aschedule)
See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more information about GitHub Actions CI.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] pabloem commented on pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
pabloem commented on pull request #16277:
URL: https://github.com/apache/beam/pull/16277#issuecomment-1006053792
LGTM
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772464841
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r774482110
##########
File path: playground/backend/internal/utils/system_utils.go
##########
@@ -16,14 +16,59 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+ "net/http"
+ "os"
+ "path/filepath"
"reflect"
"runtime"
"strings"
)
+const (
+ executableFiles = "executable_files"
Review comment:
Yes, but it is a private constant. After `[BEAM-13308]` we could use this variable like an environment variable.
##########
File path: playground/backend/internal/utils/system_utils.go
##########
@@ -16,14 +16,59 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+ "net/http"
+ "os"
+ "path/filepath"
"reflect"
"runtime"
"strings"
)
+const (
+ executableFiles = "executable_files"
Review comment:
Yes, but it is a private constant. After `[BEAM-13308]` we could use this variable as an environment variable.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772445997
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
+ baseFileFolder := filepath.Join(workingDir, "executable_files")
+ _, err := os.Stat(baseFileFolder)
+ if os.IsNotExist(err) {
+ return true
+ }
+
+ dirEntries, err := os.ReadDir(baseFileFolder)
+ if err != nil {
+ logger.Errorf("Readiness: Error during read %s: %s", baseFileFolder, err.Error())
+ return false
+ }
+
+ if len(dirEntries) >= numOfParallelJobs {
+ logger.Errorf("Readiness: Count of code processing is equals or more than possible: %d / %d", len(dirEntries), numOfParallelJobs)
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772441963
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
+ baseFileFolder := filepath.Join(workingDir, "executable_files")
Review comment:
Right now no. Everywhere we use this string value. However, we have a ticket to change this one with the value from the environment.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r774023127
##########
File path: playground/backend/internal/utils/system_utils.go
##########
@@ -16,14 +16,59 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+ "net/http"
+ "os"
+ "path/filepath"
"reflect"
"runtime"
"strings"
)
+const (
+ executableFiles = "executable_files"
+)
+
// GetFuncName returns the name of the received func
func GetFuncName(i interface{}) string {
fullName := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
splitName := strings.Split(fullName, ".")
return splitName[len(splitName)-1]
}
+
+// GetReadinessFunction returns the function that checks the readiness of the server to process a new code processing request
+func GetReadinessFunction(envs *environment.Environment) func(writer http.ResponseWriter, request *http.Request) {
+ return func(writer http.ResponseWriter, request *http.Request) {
+ if checkNumOfTheParallelJobs(envs.ApplicationEnvs.WorkingDir(), envs.BeamSdkEnvs.NumOfParallelJobs()) {
+ writer.WriteHeader(http.StatusOK)
+ } else {
+ writer.WriteHeader(http.StatusLocked)
+ }
+ }
+}
+
+// checkNumOfTheParallelJobs checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func checkNumOfTheParallelJobs(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
Review comment:
Added.
##########
File path: playground/backend/internal/utils/system_utils_test.go
##########
@@ -42,3 +45,86 @@ func TestGetFuncName(t *testing.T) {
})
}
}
+
+func Test_checkNumOfTheParallelJobs(t *testing.T) {
+ baseFileFolder := "executable_files"
+ type args struct {
+ workingDir string
+ numOfParallelJobs int
+ }
+ tests := []struct {
+ name string
+ args args
+ prepareFunc func()
+ want bool
+ }{
+ {
+ // Test case with calling checkNumOfTheParallelJobs when there is no code processing folders.
+ // As a result, want to receive true
+ name: "there is no code processing folders",
+ args: args{
+ workingDir: "",
+ numOfParallelJobs: 0,
+ },
+ prepareFunc: func() {},
+ want: true,
+ },
+ {
+ // Test case with calling checkNumOfTheParallelJobs when there is one code processing folder.
+ // As a result, want to receive true
+ name: "there is one code processing folder",
+ args: args{
+ workingDir: "",
+ numOfParallelJobs: 2,
+ },
+ prepareFunc: func() {
+ err := os.MkdirAll(filepath.Join(baseFileFolder, "1"), fs.ModePerm)
+ if err != nil {
+ panic(err)
+ }
+ },
+ want: true,
+ },
+ {
+ // Test case with calling checkNumOfTheParallelJobs when the number of the code processing folders is equals numOfParallelJobs.
+ // As a result, want to receive false
+ name: "there is one code processing folder",
+ args: args{
+ workingDir: "",
+ numOfParallelJobs: 1,
+ },
+ prepareFunc: func() {
+ err := os.MkdirAll(filepath.Join(baseFileFolder, "1"), fs.ModePerm)
+ if err != nil {
+ panic(err)
+ }
+ },
+ want: false,
+ },
+ {
+ // Test case with calling checkNumOfTheParallelJobs when the number of the code processing folders is more than numOfParallelJobs.
+ // As a result, want to receive false
+ name: "there is one code processing folder",
Review comment:
Changed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on pull request #16277:
URL: https://github.com/apache/beam/pull/16277#issuecomment-997799051
R: @KhaninArtur @ilya-kozyrev
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] KhaninArtur commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
KhaninArtur commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r773967325
##########
File path: playground/backend/internal/utils/system_utils.go
##########
@@ -16,14 +16,59 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+ "net/http"
+ "os"
+ "path/filepath"
"reflect"
"runtime"
"strings"
)
+const (
+ executableFiles = "executable_files"
+)
+
// GetFuncName returns the name of the received func
func GetFuncName(i interface{}) string {
fullName := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
splitName := strings.Split(fullName, ".")
return splitName[len(splitName)-1]
}
+
+// GetReadinessFunction returns the function that checks the readiness of the server to process a new code processing request
+func GetReadinessFunction(envs *environment.Environment) func(writer http.ResponseWriter, request *http.Request) {
+ return func(writer http.ResponseWriter, request *http.Request) {
+ if checkNumOfTheParallelJobs(envs.ApplicationEnvs.WorkingDir(), envs.BeamSdkEnvs.NumOfParallelJobs()) {
+ writer.WriteHeader(http.StatusOK)
+ } else {
+ writer.WriteHeader(http.StatusLocked)
+ }
+ }
+}
+
+// checkNumOfTheParallelJobs checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func checkNumOfTheParallelJobs(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
Review comment:
Do we have an issue ticket for this? Please, mention it here
##########
File path: playground/backend/internal/utils/system_utils_test.go
##########
@@ -42,3 +45,86 @@ func TestGetFuncName(t *testing.T) {
})
}
}
+
+func Test_checkNumOfTheParallelJobs(t *testing.T) {
+ baseFileFolder := "executable_files"
+ type args struct {
+ workingDir string
+ numOfParallelJobs int
+ }
+ tests := []struct {
+ name string
+ args args
+ prepareFunc func()
+ want bool
+ }{
+ {
+ // Test case with calling checkNumOfTheParallelJobs when there is no code processing folders.
+ // As a result, want to receive true
+ name: "there is no code processing folders",
+ args: args{
+ workingDir: "",
+ numOfParallelJobs: 0,
+ },
+ prepareFunc: func() {},
+ want: true,
+ },
+ {
+ // Test case with calling checkNumOfTheParallelJobs when there is one code processing folder.
+ // As a result, want to receive true
+ name: "there is one code processing folder",
+ args: args{
+ workingDir: "",
+ numOfParallelJobs: 2,
+ },
+ prepareFunc: func() {
+ err := os.MkdirAll(filepath.Join(baseFileFolder, "1"), fs.ModePerm)
+ if err != nil {
+ panic(err)
+ }
+ },
+ want: true,
+ },
+ {
+ // Test case with calling checkNumOfTheParallelJobs when the number of the code processing folders is equals numOfParallelJobs.
+ // As a result, want to receive false
+ name: "there is one code processing folder",
+ args: args{
+ workingDir: "",
+ numOfParallelJobs: 1,
+ },
+ prepareFunc: func() {
+ err := os.MkdirAll(filepath.Join(baseFileFolder, "1"), fs.ModePerm)
+ if err != nil {
+ panic(err)
+ }
+ },
+ want: false,
+ },
+ {
+ // Test case with calling checkNumOfTheParallelJobs when the number of the code processing folders is more than numOfParallelJobs.
+ // As a result, want to receive false
+ name: "there is one code processing folder",
Review comment:
I see you have same names for 3 tests above, let's make them different
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] pabloem merged pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
pabloem merged pull request #16277:
URL: https://github.com/apache/beam/pull/16277
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] KhaninArtur commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
KhaninArtur commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772404494
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
+ baseFileFolder := filepath.Join(workingDir, "executable_files")
+ _, err := os.Stat(baseFileFolder)
+ if os.IsNotExist(err) {
+ return true
+ }
+
+ dirEntries, err := os.ReadDir(baseFileFolder)
+ if err != nil {
+ logger.Errorf("Readiness: Error during read %s: %s", baseFileFolder, err.Error())
+ return false
+ }
+
+ if len(dirEntries) >= numOfParallelJobs {
+ logger.Errorf("Readiness: Count of code processing is equals or more than possible: %d / %d", len(dirEntries), numOfParallelJobs)
Review comment:
```suggestion
logger.Errorf("Readiness: Count of code processing is equal or more than possible: %d / %d", len(dirEntries), numOfParallelJobs)
```
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
+ baseFileFolder := filepath.Join(workingDir, "executable_files")
+ _, err := os.Stat(baseFileFolder)
+ if os.IsNotExist(err) {
+ return true
+ }
+
+ dirEntries, err := os.ReadDir(baseFileFolder)
+ if err != nil {
+ logger.Errorf("Readiness: Error during read %s: %s", baseFileFolder, err.Error())
+ return false
+ }
+
+ if len(dirEntries) >= numOfParallelJobs {
Review comment:
What happens if `numOfParallelJobs` = 0? I think we should avoid such situations
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
+ baseFileFolder := filepath.Join(workingDir, "executable_files")
Review comment:
"executable_files" seems like a constant
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ numOfParallelJobs := defaultNumOfParallelJobs
+ if value, present := os.LookupEnv(numOfParallelJobsKey); present {
+ convertedValue, err := strconv.Atoi(value)
+ if err != nil {
+ logger.Errorf("Incorrect value for %s. Should be integer. Will be used default value: %d", numOfParallelJobsKey, defaultNumOfParallelJobs)
+ } else {
+ numOfParallelJobs = convertedValue
Review comment:
In addition to the comment above, we should check, that the value is strictly positive
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ numOfParallelJobs := defaultNumOfParallelJobs
+ if value, present := os.LookupEnv(numOfParallelJobsKey); present {
+ convertedValue, err := strconv.Atoi(value)
+ if err != nil {
+ logger.Errorf("Incorrect value for %s. Should be integer. Will be used default value: %d", numOfParallelJobsKey, defaultNumOfParallelJobs)
Review comment:
@ilya-kozyrev why? I think default value is a good strategy
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772357611
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ countOfPossibleCodeProcessing := defaultCountOfPossibleCodeProcessing
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] ilya-kozyrev commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
ilya-kozyrev commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r773279996
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ numOfParallelJobs := defaultNumOfParallelJobs
+ if value, present := os.LookupEnv(numOfParallelJobsKey); present {
+ convertedValue, err := strconv.Atoi(value)
+ if err != nil {
+ logger.Errorf("Incorrect value for %s. Should be integer. Will be used default value: %d", numOfParallelJobsKey, defaultNumOfParallelJobs)
Review comment:
ok, that makes sense
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] daria-malkova commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
daria-malkova commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772321298
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ countOfPossibleCodeProcessing := defaultCountOfPossibleCodeProcessing
Review comment:
let's rename it to what Ilya proposed this morning - numOfParallelJobs
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772446962
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
+ // TODO add getting of dir executable_files from environments.
+ baseFileFolder := filepath.Join(workingDir, "executable_files")
+ _, err := os.Stat(baseFileFolder)
+ if os.IsNotExist(err) {
+ return true
+ }
+
+ dirEntries, err := os.ReadDir(baseFileFolder)
+ if err != nil {
+ logger.Errorf("Readiness: Error during read %s: %s", baseFileFolder, err.Error())
+ return false
+ }
+
+ if len(dirEntries) >= numOfParallelJobs {
Review comment:
Added check for the `numOfParallelJobs`'s value. Now if it <= 0 then the server uses the default value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772447075
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ numOfParallelJobs := defaultNumOfParallelJobs
+ if value, present := os.LookupEnv(numOfParallelJobsKey); present {
+ convertedValue, err := strconv.Atoi(value)
+ if err != nil {
+ logger.Errorf("Incorrect value for %s. Should be integer. Will be used default value: %d", numOfParallelJobsKey, defaultNumOfParallelJobs)
+ } else {
+ numOfParallelJobs = convertedValue
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] daria-malkova commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
daria-malkova commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r774460653
##########
File path: playground/backend/internal/environment/beam.go
##########
@@ -42,17 +42,23 @@ func NewExecutorConfig(compileCmd, runCmd, testCmd string, compileArgs, runArgs,
// BeamEnvs contains all environments related of ApacheBeam. These will use to run pipelines
type BeamEnvs struct {
- ApacheBeamSdk pb.Sdk
- ExecutorConfig *ExecutorConfig
- preparedModDir string
+ ApacheBeamSdk pb.Sdk
+ ExecutorConfig *ExecutorConfig
+ preparedModDir string
+ numOfParallelJobs int
}
// NewBeamEnvs is a BeamEnvs constructor
-func NewBeamEnvs(apacheBeamSdk pb.Sdk, executorConfig *ExecutorConfig, preparedModDir string) *BeamEnvs {
- return &BeamEnvs{ApacheBeamSdk: apacheBeamSdk, ExecutorConfig: executorConfig, preparedModDir: preparedModDir}
+func NewBeamEnvs(apacheBeamSdk pb.Sdk, executorConfig *ExecutorConfig, preparedModDir string, numOfParallelJobs int) *BeamEnvs {
+ return &BeamEnvs{ApacheBeamSdk: apacheBeamSdk, ExecutorConfig: executorConfig, preparedModDir: preparedModDir, numOfParallelJobs: numOfParallelJobs}
}
// PreparedModDir returns the path to the directory where prepared go.mod and go.sum are located
func (b *BeamEnvs) PreparedModDir() string {
return b.preparedModDir
}
+
+// NumOfParallelJobs returns the max number of the possible code processing for the SDK on that instance at the same time
Review comment:
// NumOfParallelJobs returns the max number of the possible code executions on the instance simultaneously.
##########
File path: playground/backend/internal/utils/system_utils.go
##########
@@ -16,14 +16,59 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+ "net/http"
+ "os"
+ "path/filepath"
"reflect"
"runtime"
"strings"
)
+const (
+ executableFiles = "executable_files"
Review comment:
Don't we already have this constant somewhere?
##########
File path: playground/backend/internal/utils/system_utils.go
##########
@@ -16,14 +16,59 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+ "net/http"
+ "os"
+ "path/filepath"
"reflect"
"runtime"
"strings"
)
+const (
+ executableFiles = "executable_files"
+)
+
// GetFuncName returns the name of the received func
func GetFuncName(i interface{}) string {
fullName := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
splitName := strings.Split(fullName, ".")
return splitName[len(splitName)-1]
}
+
+// GetReadinessFunction returns the function that checks the readiness of the server to process a new code processing request
+func GetReadinessFunction(envs *environment.Environment) func(writer http.ResponseWriter, request *http.Request) {
+ return func(writer http.ResponseWriter, request *http.Request) {
+ if checkNumOfTheParallelJobs(envs.ApplicationEnvs.WorkingDir(), envs.BeamSdkEnvs.NumOfParallelJobs()) {
+ writer.WriteHeader(http.StatusOK)
+ } else {
+ writer.WriteHeader(http.StatusLocked)
+ }
+ }
+}
+
+// checkNumOfTheParallelJobs checks the number of already working code processing.
Review comment:
of already -> of currently
code processing -> code executions
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772439908
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ numOfParallelJobs := defaultNumOfParallelJobs
+ if value, present := os.LookupEnv(numOfParallelJobsKey); present {
+ convertedValue, err := strconv.Atoi(value)
+ if err != nil {
+ logger.Errorf("Incorrect value for %s. Should be integer. Will be used default value: %d", numOfParallelJobsKey, defaultNumOfParallelJobs)
Review comment:
Yes here I try to get this value from the environment, but if it doesn't exist or if it is set up with an incorrect value then I just log it and use the default value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] ilya-kozyrev commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
ilya-kozyrev commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772381746
##########
File path: playground/backend/internal/environment/environment_service.go
##########
@@ -137,6 +140,17 @@ func GetNetworkEnvsFromOsEnvs() (*NetworkEnvs, error) {
func ConfigureBeamEnvs(workDir string) (*BeamEnvs, error) {
sdk := pb.Sdk_SDK_UNSPECIFIED
preparedModDir, modDirExist := os.LookupEnv(preparedModDirKey)
+
+ numOfParallelJobs := defaultNumOfParallelJobs
+ if value, present := os.LookupEnv(numOfParallelJobsKey); present {
+ convertedValue, err := strconv.Atoi(value)
+ if err != nil {
+ logger.Errorf("Incorrect value for %s. Should be integer. Will be used default value: %d", numOfParallelJobsKey, defaultNumOfParallelJobs)
Review comment:
I think we can't run the server without this variable, at least it should return this error.
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -19,12 +19,28 @@ import (
"beam.apache.org/playground/backend/internal/logger"
"context"
"net/http"
+ "os"
+ "path/filepath"
)
// listenHttp binds the http.Handler on the TCP network address
-func listenHttp(ctx context.Context, errChan chan error, envs environment.NetworkEnvs, handler http.Handler) {
- logger.Infof("listening HTTP at %s\n", envs.Address())
- if err := http.ListenAndServe(envs.Address(), handler); err != nil {
+func listenHttp(ctx context.Context, errChan chan error, envs *environment.Environment, handler http.Handler) {
+ address := envs.NetworkEnvs.Address()
+ logger.Infof("listening HTTP at %s\n", address)
+
+ mux := http.NewServeMux()
+ mux.Handle("/", handler)
+ mux.HandleFunc("/readiness", func(w http.ResponseWriter, r *http.Request) {
Review comment:
Could you please extract the function to logical block?
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -33,3 +49,28 @@ func listenHttp(ctx context.Context, errChan chan error, envs environment.Networ
return
}
}
+
+// isReady checks the number of already working code processing.
+// It counts by the number of the /path/to/workingDir/executable_files/{pipelineId} folders.
+// If it is equals or more than numOfParallelJobs, then returns false.
+// If it is less than numOfParallelJobs, then returns true.
+func isReady(workingDir string, numOfParallelJobs int) bool {
Review comment:
I believe that server/http not a best place for business logic
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r772464747
##########
File path: playground/backend/cmd/server/http.go
##########
@@ -19,12 +19,28 @@ import (
"beam.apache.org/playground/backend/internal/logger"
"context"
"net/http"
+ "os"
+ "path/filepath"
)
// listenHttp binds the http.Handler on the TCP network address
-func listenHttp(ctx context.Context, errChan chan error, envs environment.NetworkEnvs, handler http.Handler) {
- logger.Infof("listening HTTP at %s\n", envs.Address())
- if err := http.ListenAndServe(envs.Address(), handler); err != nil {
+func listenHttp(ctx context.Context, errChan chan error, envs *environment.Environment, handler http.Handler) {
+ address := envs.NetworkEnvs.Address()
+ logger.Infof("listening HTTP at %s\n", address)
+
+ mux := http.NewServeMux()
+ mux.Handle("/", handler)
+ mux.HandleFunc("/readiness", func(w http.ResponseWriter, r *http.Request) {
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [beam] AydarZaynutdinov commented on a change in pull request #16277: [BEAM-13124][Playground] Create readiness endpoint
Posted by GitBox <gi...@apache.org>.
AydarZaynutdinov commented on a change in pull request #16277:
URL: https://github.com/apache/beam/pull/16277#discussion_r774482879
##########
File path: playground/backend/internal/utils/system_utils.go
##########
@@ -16,14 +16,59 @@
package utils
import (
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/logger"
+ "net/http"
+ "os"
+ "path/filepath"
"reflect"
"runtime"
"strings"
)
+const (
+ executableFiles = "executable_files"
+)
+
// GetFuncName returns the name of the received func
func GetFuncName(i interface{}) string {
fullName := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
splitName := strings.Split(fullName, ".")
return splitName[len(splitName)-1]
}
+
+// GetReadinessFunction returns the function that checks the readiness of the server to process a new code processing request
+func GetReadinessFunction(envs *environment.Environment) func(writer http.ResponseWriter, request *http.Request) {
+ return func(writer http.ResponseWriter, request *http.Request) {
+ if checkNumOfTheParallelJobs(envs.ApplicationEnvs.WorkingDir(), envs.BeamSdkEnvs.NumOfParallelJobs()) {
+ writer.WriteHeader(http.StatusOK)
+ } else {
+ writer.WriteHeader(http.StatusLocked)
+ }
+ }
+}
+
+// checkNumOfTheParallelJobs checks the number of already working code processing.
Review comment:
Done.
##########
File path: playground/backend/internal/environment/beam.go
##########
@@ -42,17 +42,23 @@ func NewExecutorConfig(compileCmd, runCmd, testCmd string, compileArgs, runArgs,
// BeamEnvs contains all environments related of ApacheBeam. These will use to run pipelines
type BeamEnvs struct {
- ApacheBeamSdk pb.Sdk
- ExecutorConfig *ExecutorConfig
- preparedModDir string
+ ApacheBeamSdk pb.Sdk
+ ExecutorConfig *ExecutorConfig
+ preparedModDir string
+ numOfParallelJobs int
}
// NewBeamEnvs is a BeamEnvs constructor
-func NewBeamEnvs(apacheBeamSdk pb.Sdk, executorConfig *ExecutorConfig, preparedModDir string) *BeamEnvs {
- return &BeamEnvs{ApacheBeamSdk: apacheBeamSdk, ExecutorConfig: executorConfig, preparedModDir: preparedModDir}
+func NewBeamEnvs(apacheBeamSdk pb.Sdk, executorConfig *ExecutorConfig, preparedModDir string, numOfParallelJobs int) *BeamEnvs {
+ return &BeamEnvs{ApacheBeamSdk: apacheBeamSdk, ExecutorConfig: executorConfig, preparedModDir: preparedModDir, numOfParallelJobs: numOfParallelJobs}
}
// PreparedModDir returns the path to the directory where prepared go.mod and go.sum are located
func (b *BeamEnvs) PreparedModDir() string {
return b.preparedModDir
}
+
+// NumOfParallelJobs returns the max number of the possible code processing for the SDK on that instance at the same time
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org