You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fc...@apache.org on 2025/10/03 17:02:17 UTC
(flink-connector-http) branch main updated (0c46b8e -> 645c950)
This is an automated email from the ASF dual-hosted git repository.
fcsaky pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-http.git
from 0c46b8e [FLIP-532] Initialize repo
add 8bcc01f Init commit
add 01fd4e6 Create LICENCE
add 30b7b7d Create CONTRIBUTING.md
add bd1378c Create CHANGELOG.md
add c8e002e Add Run Example to README.md
add 0509f52 Remove log4j2.properties from src/main folder.
add 5e05558 Update README.md
add 4ea192e Update README.md
add ad1a003 flink upgrade to 1.15
add e79a21b update readme
add 0c06999 upgrade log4j
add 858bf81 Add build Workflow
add e6c8b47 Modify pom.xml to conform with OSSRH requirements
add d81bccf Prepare rest of the GitHub workflows
add 440b45b Fix CI/CD for prepare release (#3)
add 28dc735 Bump CHANGELOG for release 0.1.0
add 5425c62 Bump development version to 0.2.0-SNAPSHOT
add 990c9b7 Attach Maven sources
add 27b2b42 Add initial `HttpSink` implementation (#4)
add 7543b24 Http Sink Table API (#5)
add b8b9ba7 Disable retries of failed requests (#6)
add 1f8cf65 Bump version to 0.3.0-SNAPSHOT (#7)
add ccb6387 Package_refactoring - add internal package and expose public API
add 72c5c8c checkstyle - add checkstyle based on delta.io/connectors checkstyle
add 465fe74 ReadmeUpdate - add Maven link to readme.
add 0de047e Add Maven badge to README
add 5212974 TestPackageRefactoring - refactoring test package to match source folder. Fix checkstyle issues.
add 18e7955 ESP-98ESP-98_SinkParameters - pass headers via properties to Http client
add 9805699 ESP-98ESP-98_SinkParameters - add unit tests
add 2829ef6 ESP-98ESP-98_SinkParameters - add unit tests
add 762e2d0 ESP-98ESP-98_SinkParameters - add properties to Table API.
add 5d08435 ESP-98ESP-98_SinkParameters - add properties to Table API.
add 3311e62 ESP-98ESP-98_SinkParameters - CHANGELOG.md and README.md update
add 4b16f2c ESP-98ESP-98_SinkParameters - changes after code review
add f3a5cdc Bump development version after release of 0.3.0 (#13)
add 97b9e33 Fix JavaDoc errors (#14)
add 762b13e Add JavaDoc.io badge (#15)
add c58277e ESP-99_ErrorCodes - Error Code configuration from properties. (#16)
add a7dc4f6 Drop Dependency for Apache HttpClient (#17)
add f21d2e5 ESP-148_UseFlink_JsonFormat_InSource - Use Flink Json Format instead custom implemented Json Format for Source Lookup. (#19)
add 89065e2 LookupSource_TestRename - Source Test name change + remove unused method argument in source factory. (#20)
add 0d366de 1ESP-119_Http_headers_SourceSupport - Add Http header support for Http Lookup Source. (#21)
add 80a8935 ESP-171_HttpLookupSource_ErrorCodeSupport - HTTP ErrorCode Status config for Lookup Source (#22)
add 87c9a8a Introduce custom HTTP Sink Post Request Callback and its Factory (#23)
add ae8b3dd Bump development version after release of 0.4.0 (#24)
add 5c82a84 ESP-117 adding tls and mTLS support for Http Connector (#25)
add cc7d58c ESP-221_BasicAuth - adding HeaderPreprocessor and support for Basic Authentication mechanism. (#26)
add 39921c2 Introduce LookupQueryCreator and its Factory
add c504ac9 Increase the coverage a bit
add a7d0143 Add ElasticSearchLiteQueryCreator
add dd167a0 Bump CHANGELOG for release 0.5.0
add e654078 Bump development version to 0.6.0-SNAPSHOT
add 5a861a6 ESP-263 - Adding body based REST methods to Lookup Source.
add 8fc8c1a Bump development version after release of 0.6.0 (#30)
add 9941292 [HTTP-34] Add support for lookup on Row type (#35)
add 25b959f Bump development version after release of 0.7.0 (#36)
add 01ccca9 [HTTP-38] - Fixing issue with handling Http timeout where Event task was not removed from Flink's Operator AsyncIO internal queue.
add 5c55729 Bump CHANGELOG for release 0.8.0
add d9f7cd3 Bump development version to 0.9.0-SNAPSHOT
add a2bfb67 [HTTP-43] - Remove Encoding format from HttpLookupTableSource
add 77d8167 Fix typo in examples in README.md.
add 4bdb5a5 [HTTP-45] - Fix issue with not printing HttpRequest body/parameters for Lookup Source by Slf4JHttpLookupPostRequestCallback.
add f665da4 Prepare 0.8.1 hotfix version
add 3c8397a Bump CHANGELOG for release 0.8.1
add 1ee6a72 Bump development version to 0.9.0-SNAPSHOT
add e20f492 Update README.md TODO list
add 537b689 [HTTP-37] - Add support for Flink 1.16. Modify CI/CD to run builds for Flink 1.15 and 1.16.
add 29ee64b Bump CHANGELOG for release 0.9.0
add c0c3f57 Bump development version to 0.10.0-SNAPSHOT
add d3c438a rc-0.9.0_testRunFix - fix publish build #2
add 8baed99 Update README.md
add b485a64 Cleaning unused gitlab ci definition (#56)
add 99365f3 HTTP-42 Add support for Batch request submission in HTTP Sink. (#58)
add 4209cd4 Bump development version after release of 0.10.0 (#60)
add 1b2b164 Update README.md - how to use request per element mode in Http Sink
add 17d0621 Update README.md
add 02cf022 main - add test for passing properties to format used by lookup query creator.
add b4614c0 Issue54 Fix class loader and Prefixed Config option issues (#68)
add 58e7b46 Bump development version after release of 0.11.0 (#70)
add 99d7f2b HTTP-76 Allow enforcing the use of raw Authorization header (#77)
add 61e6ca7 HTTP-78 Support for optional query parameters for body-based requests (#79)
add c6683d7 Bump vsoch/pull-request-action version (#81)
add 2952b73 Bump development version after release of 0.12.0 (#82)
add 43786bc [HTTP-64] move to junit 5 and fix junit failures (#72)
add 93a6a9d HTTP-74 Add path parameter support (#87)
add 91fac5a HTTP-74 Add path parameter support (#84)
add 27b0f61 Bump development version after release of 0.13.0 (#88)
add feeabd5 HTTP-92 Customization of HTTP lookup source logger
add 09286ef Bump development version after release of 0.14.0 (#95)
add c74f09e HTTP-96 Fix logging code in JavaNetHttpPollingClient (#97)
add 959e60e Fix jacoco code coverage info in PR. Remove printing jacoco stat to console (#101)
add 23ba94f Reinstatiate javadoc step in common build (#102)
add 5fc9341 PR from forked repo fix (#105)
add 48ae840 HTTP-63 Add caching (#109)
add 44e30db Bump development version after release of 0.15.0 (#110)
add b391b3e update publish to maven setup (#112)
add 91958ee Maven publish fix (#113)
add 01c6217 Maven publish fix (#114)
add a9e21f3 Maven publish fix (#115)
add 45643b6 Maven publish fix (#116)
add e38d73d Update publish.yml
add 1e62389 Update publish.yml
add f026603 maven-publish-fix - fixing maven central publish
add d65de80 HTTP-91 Bearer token support (#117)
add 75a469d Fix adding coverage to PRs (#131)
add ea9e9f4 HTTP-121 Ensure SerializationSchema is used in thread-safe way (#127)
add de8096b Add missing tests (#130)
add d32368d HTTP-119 Pickup up public CAs when there are no security options configured (#128)
add 350c62f Bump CHANGELOG for release 0.16.0
add 85cf545 Bump development version to 0.17.0-SNAPSHOT
add eb8995a Support source lookup array results (#135)
add 202d358 Bump CHANGELOG for release 0.17.0
add 1ca7b4c Bump development version to 0.18.0-SNAPSHOT
add a0db115 HTTP-138 ignore Eclipse files in .gitignore (#139)
add cb93c76 HTTP-103 Support Flink 1.20 (#140)
add 2ea462e Bump CHANGELOG for release 0.18.0
add 203c8e3 Bump development version to 0.19.0-SNAPSHOT
add b32c518 HTTP-145 SQL explain drives rest flow to OIDC endpoint (#146)
add be88683 Bump CHANGELOG for release 0.19.0
add 849a97f Bump development version to 0.20.0-SNAPSHOT
add 4c7d376 HTTP-122 Retry for source lookup table (#148)
add 869b275 HTTP-99 Add support for generic json and URL query creator (#149)
add 444ea89 HTTP-150 Add Proxy to HttpClient for Lookup Source (#151)
add 99a1e52 Add missing permission in prepare_release_branch (#152)
add 19dd386 Bump CHANGELOG for release 0.20.0
add 4e395c5 Bump development version to 0.21.0-SNAPSHOT
new 645c950 [FLINK-38245] Initial contribution of Getindata connector
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.checkstyle | 15 +
.github/boring-cyborg.yml | 86 ++
.github/workflows/push_pr.yml | 37 +
.github/workflows/weekly.yml | 41 +
.gitignore | 37 +
.mvn/jvm.config | 7 +
.mvn/wrapper/maven-wrapper.properties | 22 +
CHANGELOG.md | 285 +++++
NOTICE | 17 +
README.md | 68 ++
docs/content/docs/datastream/http.md | 179 ++++
docs/content/docs/table/http.md | 545 ++++++++++
flink-connector-http/pom.xml | 465 ++++++++
.../connector/http/HttpPostRequestCallback.java | 46 +
.../http/HttpPostRequestCallbackFactory.java | 79 ++
.../org/apache/flink/connector/http/HttpSink.java | 90 ++
.../flink/connector/http/HttpSinkBuilder.java | 211 ++++
.../HttpStatusCodeValidationFailedException.java | 33 +
.../org/apache/flink/connector/http/LookupArg.java | 33 +
.../flink/connector/http/LookupQueryCreator.java | 43 +
.../connector/http/LookupQueryCreatorFactory.java | 68 ++
.../http/SchemaLifecycleAwareElementConverter.java | 50 +
.../http/auth/OidcAccessTokenManager.java | 155 +++
.../connector/http/clients/PollingClient.java | 42 +
.../http/clients/PollingClientFactory.java | 36 +
.../connector/http/clients/SinkHttpClient.java | 43 +
.../http/clients/SinkHttpClientBuilder.java | 40 +
.../http/clients/SinkHttpClientResponse.java | 42 +
.../connector/http/config/ConfigException.java | 52 +
.../http/config/HttpConnectorConfigConstants.java | 159 +++
.../http/config/SinkRequestSubmitMode.java | 34 +
.../BasicAuthHeaderValuePreprocessor.java | 72 ++
.../preprocessor/ComposeHeaderPreprocessor.java | 60 ++
.../http/preprocessor/HeaderPreprocessor.java | 33 +
.../http/preprocessor/HeaderValuePreprocessor.java | 35 +
.../OIDCAuthHeaderValuePreprocessor.java | 64 ++
.../connector/http/retry/HttpClientWithRetry.java | 93 ++
.../connector/http/retry/RetryConfigProvider.java | 75 ++
.../connector/http/retry/RetryStrategyType.java | 45 +
.../connector/http/security/SecurityContext.java | 244 +++++
.../http/security/SelfSignedTrustManager.java | 71 ++
.../connector/http/sink/HttpSinkInternal.java | 202 ++++
.../connector/http/sink/HttpSinkRequestEntry.java | 48 +
.../flink/connector/http/sink/HttpSinkWriter.java | 161 +++
.../http/sink/HttpSinkWriterStateSerializer.java | 53 +
.../sink/httpclient/AbstractRequestSubmitter.java | 64 ++
.../sink/httpclient/BatchRequestSubmitter.java | 153 +++
.../httpclient/BatchRequestSubmitterFactory.java | 89 ++
.../http/sink/httpclient/HttpRequest.java | 33 +
.../httpclient/JavaNetHttpResponseWrapper.java | 46 +
.../sink/httpclient/JavaNetSinkHttpClient.java | 137 +++
.../PerRequestRequestSubmitterFactory.java | 50 +
.../http/sink/httpclient/PerRequestSubmitter.java | 94 ++
.../http/sink/httpclient/RequestSubmitter.java | 30 +
.../sink/httpclient/RequestSubmitterFactory.java | 26 +
.../http/status/ComposeHttpStatusCodeChecker.java | 176 +++
.../connector/http/status/HttpCodesParser.java | 85 ++
.../connector/http/status/HttpResponseChecker.java | 78 ++
.../http/status/HttpResponseCodeType.java | 63 ++
.../http/status/HttpStatusCodeChecker.java | 34 +
.../status/IncludeListHttpStatusCodeChecker.java | 39 +
.../status/SingleValueHttpStatusCodeChecker.java | 44 +
.../http/status/TypeStatusCodeChecker.java | 61 ++
.../table/SerializationSchemaElementConverter.java | 61 ++
.../table/lookup/AsyncHttpTableLookupFunction.java | 131 +++
.../http/table/lookup/BodyBasedRequestFactory.java | 88 ++
.../http/table/lookup/GetRequestFactory.java | 90 ++
.../http/table/lookup/HttpLookupConfig.java | 48 +
.../table/lookup/HttpLookupConnectorOptions.java | 200 ++++
.../table/lookup/HttpLookupSourceRequestEntry.java | 47 +
.../http/table/lookup/HttpLookupTableSource.java | 258 +++++
.../table/lookup/HttpLookupTableSourceFactory.java | 232 ++++
.../http/table/lookup/HttpRequestFactory.java | 35 +
.../http/table/lookup/HttpTableLookupFunction.java | 88 ++
.../table/lookup/JavaNetHttpPollingClient.java | 252 +++++
.../lookup/JavaNetHttpPollingClientFactory.java | 46 +
.../http/table/lookup/LookupQueryInfo.java | 86 ++
.../connector/http/table/lookup/LookupRow.java | 72 ++
.../http/table/lookup/LookupSchemaEntry.java | 43 +
.../http/table/lookup/RequestFactoryBase.java | 134 +++
.../table/lookup/RowDataLookupSchemaEntryBase.java | 48 +
.../RowDataSingleValueLookupSchemaEntry.java | 93 ++
.../table/lookup/RowTypeLookupSchemaEntry.java | 102 ++
.../lookup/Slf4JHttpLookupPostRequestCallback.java | 84 ++
.../Slf4jHttpLookupPostRequestCallbackFactory.java | 53 +
.../http/table/lookup/TableSourceHelper.java | 73 ++
.../ElasticSearchLiteQueryCreator.java | 69 ++
.../ElasticSearchLiteQueryCreatorFactory.java | 56 +
.../querycreators/GenericGetQueryCreator.java | 58 +
.../GenericGetQueryCreatorFactory.java | 56 +
.../GenericJsonAndUrlQueryCreator.java | 285 +++++
.../GenericJsonAndUrlQueryCreatorFactory.java | 157 +++
.../querycreators/GenericJsonQueryCreator.java | 71 ++
.../GenericJsonQueryCreatorFactory.java | 92 ++
.../lookup/querycreators/ObjectMapperAdapter.java | 42 +
.../lookup/querycreators/PrefixedConfigOption.java | 102 ++
.../QueryFormatAwareConfiguration.java | 59 ++
.../connector/http/table/sink/HttpDynamicSink.java | 277 +++++
.../sink/HttpDynamicSinkConnectorOptions.java | 44 +
.../table/sink/HttpDynamicTableSinkFactory.java | 116 ++
.../table/sink/Slf4jHttpPostRequestCallback.java | 69 ++
.../sink/Slf4jHttpPostRequestCallbackFactory.java | 53 +
.../flink/connector/http/utils/ConfigUtils.java | 132 +++
.../connector/http/utils/HttpHeaderUtils.java | 150 +++
.../http/utils/JavaNetHttpClientFactory.java | 235 ++++
.../flink/connector/http/utils/ProxyConfig.java | 61 ++
.../http/utils/SerializationSchemaUtils.java | 67 ++
.../utils/SynchronizedSerializationSchema.java | 52 +
.../flink/connector/http/utils/ThreadUtils.java | 37 +
.../connector/http/utils/uri/CharArrayBuffer.java | 117 ++
.../connector/http/utils/uri/NameValuePair.java | 43 +
.../connector/http/utils/uri/ParserCursor.java | 72 ++
.../connector/http/utils/uri/TokenParser.java | 135 +++
.../flink/connector/http/utils/uri/URIBuilder.java | 172 +++
.../connector/http/utils/uri/URLEncodedUtils.java | 437 ++++++++
.../org.apache.flink.table.factories.Factory | 28 +
.../http/HttpPostRequestCallbackFactoryTest.java | 194 ++++
.../connector/http/HttpsConnectionTestBase.java | 62 ++
.../flink/connector/http/StreamTableJob.java | 77 ++
.../apache/flink/connector/http/TestHelper.java | 65 ++
.../http/TestLookupPostRequestCallbackFactory.java | 53 +
.../http/TestPostRequestCallbackFactory.java | 51 +
.../http/WireMockServerPortAllocator.java | 36 +
.../flink/connector/http/app/HttpStubApp.java | 57 +
.../flink/connector/http/app/JsonTransform.java | 125 +++
.../http/auth/OidcAccessTokenManagerTest.java | 257 +++++
.../connector/http/config/ConfigExceptionTest.java | 40 +
.../http/retry/HttpClientWithRetryTest.java | 171 +++
.../http/retry/RetryConfigProviderTest.java | 89 ++
.../http/retry/RetryStrategyTypeTest.java | 61 ++
.../connector/http/sink/HttpSinkBuilderTest.java | 93 ++
.../http/sink/HttpSinkConnectionTest.java | 378 +++++++
.../sink/HttpSinkWriterStateSerializerTest.java | 52 +
.../connector/http/sink/HttpSinkWriterTest.java | 114 ++
.../BatchRequestSubmitterFactoryTest.java | 96 ++
.../sink/httpclient/BatchRequestSubmitterTest.java | 103 ++
.../JavaNetSinkHttpClientConnectionTest.java | 329 ++++++
.../sink/httpclient/JavaNetSinkHttpClientTest.java | 138 +++
.../status/ComposeHttpStatusCodeCheckerTest.java | 181 ++++
.../connector/http/status/HttpCodesParserTest.java | 105 ++
.../http/status/HttpResponseCheckerTest.java | 127 +++
.../BasicAuthHeaderValuePreprocessorTest.java | 46 +
.../http/table/ComposeHeaderPreprocessorTest.java | 43 +
.../lookup/AsyncHttpTableLookupFunctionTest.java | 238 +++++
.../table/lookup/BodyBasedRequestFactoryTest.java | 145 +++
.../lookup/HttpLookupTableSourceFactoryTest.java | 139 +++
.../lookup/HttpLookupTableSourceITCaseTest.java | 1118 ++++++++++++++++++++
.../table/lookup/HttpLookupTableSourceTest.java | 258 +++++
.../JavaNetHttpPollingClientConnectionTest.java | 542 ++++++++++
.../JavaNetHttpPollingClientFactoryTest.java | 51 +
...avaNetHttpPollingClientHttpsConnectionTest.java | 363 +++++++
.../table/lookup/JavaNetHttpPollingClientTest.java | 237 +++++
.../JavaNetHttpPollingClientWithWireTest.java | 191 ++++
.../connector/http/table/lookup/JsonTransform.java | 126 +++
.../http/table/lookup/LookupQueryInfoTest.java | 107 ++
.../RowDataSingleValueLookupSchemaEntryTest.java | 47 +
.../table/lookup/RowTypeLookupSchemaEntryTest.java | 177 ++++
.../http/table/lookup/TableSourceHelperTest.java | 53 +
.../lookup/querycreators/CustomFormatFactory.java | 66 ++
.../querycreators/CustomJsonFormatFactory.java | 67 ++
.../ElasticSearchLiteQueryCreatorTest.java | 158 +++
.../querycreators/GenericGetQueryCreatorTest.java | 208 ++++
.../GenericJsonAndUrlQueryCreatorFactoryTest.java | 138 +++
.../GenericJsonAndUrlQueryCreatorTest.java | 267 +++++
.../GenericJsonQueryCreatorFactoryTest.java | 111 ++
.../querycreators/GenericJsonQueryCreatorTest.java | 92 ++
.../table/lookup/querycreators/PersonBean.java | 28 +
.../lookup/querycreators/QueryCreatorUtils.java | 76 ++
.../QueryFormatAwareConfigurationTest.java | 59 ++
.../BatchRequestHttpDynamicSinkInsertTest.java | 364 +++++++
.../http/table/sink/HttpDynamicSinkTest.java | 134 +++
.../sink/HttpDynamicTableSinkFactoryTest.java | 113 ++
.../sink/PerRequestHttpDynamicSinkInsertTest.java | 331 ++++++
.../connector/http/utils/ConfigUtilsTest.java | 292 +++++
.../connector/http/utils/HttpHeaderUtilsTest.java | 47 +
.../http/utils/JavaNetHttpClientFactoryTest.java | 141 +++
.../http/utils/uri/CharArrayBufferTest.java | 96 ++
.../connector/http/utils/uri/ParserCursorTest.java | 49 +
.../connector/http/utils/uri/TokenParserTest.java | 41 +
.../connector/http/utils/uri/URIBuilderTest.java | 298 ++++++
.../http/utils/uri/URLEncodedUtilsTest.java | 125 +++
.../org.apache.flink.table.factories.Factory | 24 +
.../src/test/resources/auth/AuthResult.json | 4 +
.../http-array-result-with-nulls/HttpResult.json | 20 +
.../resources/http-array-result/HttpResult.json | 36 +
.../src/test/resources/http/HttpResult.json | 51 +
.../src/test/resources/json/sink/allInOneBatch.txt | 47 +
.../resources/json/sink/fourSingleEventBatches.txt | 60 ++
.../src/test/resources/json/sink/threeBatches.txt | 53 +
.../src/test/resources/json/sink/twoBatches.txt | 50 +
.../src/test/resources/security/certs/ca.crt | 34 +
.../src/test/resources/security/certs/ca.key | 51 +
.../src/test/resources/security/certs/ca.pass.key | 54 +
.../src/test/resources/security/certs/ca.srl | 1 +
.../security/certs/ca_server_bundle.cert.pem | 66 ++
.../src/test/resources/security/certs/client.crt | 32 +
.../src/test/resources/security/certs/client.csr | 29 +
.../src/test/resources/security/certs/client.key | 51 +
.../test/resources/security/certs/client.pass.key | 54 +
.../resources/security/certs/clientPrivateKey.der | Bin 0 -> 2375 bytes
.../resources/security/certs/clientPrivateKey.pem | 52 +
.../resources/security/certs/client_keyStore.p12 | Bin 0 -> 4181 bytes
.../src/test/resources/security/certs/server.crt | 32 +
.../src/test/resources/security/certs/server.csr | 29 +
.../src/test/resources/security/certs/server.key | 51 +
.../test/resources/security/certs/server.pass.key | 54 +
.../resources/security/certs/serverKeyStore.jks | Bin 0 -> 4383 bytes
.../resources/security/certs/serverTrustStore.jks | Bin 0 -> 4383 bytes
flink-sql-connector-http/pom.xml | 147 +++
.../src/main/resources/META-INF/NOTICE | 12 +
.../flink/connector/http/PackagingITCase.java | 42 +
lombok.config | 1 +
pom.xml | 611 +++++++++++
tools/ci/log4j.properties | 72 ++
tools/maven/checkstyle.xml | 564 ++++++++++
tools/maven/suppressions.xml | 27 +
216 files changed, 23973 insertions(+)
create mode 100644 .checkstyle
create mode 100644 .github/boring-cyborg.yml
create mode 100644 .github/workflows/push_pr.yml
create mode 100644 .github/workflows/weekly.yml
create mode 100644 .gitignore
create mode 100644 .mvn/jvm.config
create mode 100644 .mvn/wrapper/maven-wrapper.properties
create mode 100644 CHANGELOG.md
create mode 100644 NOTICE
create mode 100644 README.md
create mode 100644 docs/content/docs/datastream/http.md
create mode 100644 docs/content/docs/table/http.md
create mode 100644 flink-connector-http/pom.xml
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallback.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSink.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSinkBuilder.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpStatusCodeValidationFailedException.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupArg.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreator.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreatorFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/SchemaLifecycleAwareElementConverter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/auth/OidcAccessTokenManager.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClient.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClientFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClient.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientBuilder.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientResponse.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/config/ConfigException.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/config/HttpConnectorConfigConstants.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/config/SinkRequestSubmitMode.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/BasicAuthHeaderValuePreprocessor.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/ComposeHeaderPreprocessor.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderPreprocessor.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderValuePreprocessor.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/OIDCAuthHeaderValuePreprocessor.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/HttpClientWithRetry.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryConfigProvider.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryStrategyType.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SecurityContext.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SelfSignedTrustManager.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkInternal.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkRequestEntry.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriterStateSerializer.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/AbstractRequestSubmitter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitterFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/HttpRequest.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetHttpResponseWrapper.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetSinkHttpClient.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestRequestSubmitterFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestSubmitter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitterFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/ComposeHttpStatusCodeChecker.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpCodesParser.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseChecker.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseCodeType.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpStatusCodeChecker.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/IncludeListHttpStatusCodeChecker.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/SingleValueHttpStatusCodeChecker.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/status/TypeStatusCodeChecker.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/SerializationSchemaElementConverter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/AsyncHttpTableLookupFunction.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/BodyBasedRequestFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/GetRequestFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConfig.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupSourceRequestEntry.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSource.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpRequestFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpTableLookupFunction.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClient.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupQueryInfo.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupRow.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupSchemaEntry.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RequestFactoryBase.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataLookupSchemaEntryBase.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataSingleValueLookupSchemaEntry.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowTypeLookupSchemaEntry.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4JHttpLookupPostRequestCallback.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4jHttpLookupPostRequestCallbackFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/TableSourceHelper.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreator.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreatorFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreator.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreatorFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonAndUrlQueryCreatorFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonQueryCreator.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonQueryCreatorFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ObjectMapperAdapter.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/PrefixedConfigOption.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/QueryFormatAwareConfiguration.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSink.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicSinkConnectorOptions.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/Slf4jHttpPostRequestCallback.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/table/sink/Slf4jHttpPostRequestCallbackFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/ConfigUtils.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/HttpHeaderUtils.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/JavaNetHttpClientFactory.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/ProxyConfig.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/SerializationSchemaUtils.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/SynchronizedSerializationSchema.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/ThreadUtils.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/uri/CharArrayBuffer.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/uri/NameValuePair.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/uri/ParserCursor.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/uri/TokenParser.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/uri/URIBuilder.java
create mode 100644 flink-connector-http/src/main/java/org/apache/flink/connector/http/utils/uri/URLEncodedUtils.java
create mode 100644 flink-connector-http/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/HttpsConnectionTestBase.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/StreamTableJob.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/TestHelper.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/TestLookupPostRequestCallbackFactory.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/TestPostRequestCallbackFactory.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/WireMockServerPortAllocator.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/app/HttpStubApp.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/app/JsonTransform.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/auth/OidcAccessTokenManagerTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/config/ConfigExceptionTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/retry/HttpClientWithRetryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/retry/RetryConfigProviderTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/retry/RetryStrategyTypeTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/HttpSinkBuilderTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/HttpSinkConnectionTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/HttpSinkWriterStateSerializerTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/HttpSinkWriterTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitterFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitterTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/httpclient/JavaNetSinkHttpClientConnectionTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/httpclient/JavaNetSinkHttpClientTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/sink/httpclient/status/ComposeHttpStatusCodeCheckerTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/status/HttpCodesParserTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/status/HttpResponseCheckerTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/BasicAuthHeaderValuePreprocessorTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/ComposeHeaderPreprocessorTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/AsyncHttpTableLookupFunctionTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/BodyBasedRequestFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceITCaseTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientConnectionTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientHttpsConnectionTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientWithWireTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/JsonTransform.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/LookupQueryInfoTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/RowDataSingleValueLookupSchemaEntryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/RowTypeLookupSchemaEntryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/TableSourceHelperTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/CustomFormatFactory.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/CustomJsonFormatFactory.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreatorTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreatorTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonAndUrlQueryCreatorFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonAndUrlQueryCreatorTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonQueryCreatorFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonQueryCreatorTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/PersonBean.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/QueryCreatorUtils.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/lookup/querycreators/QueryFormatAwareConfigurationTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/BatchRequestHttpDynamicSinkInsertTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/HttpDynamicSinkTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/HttpDynamicTableSinkFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/table/sink/PerRequestHttpDynamicSinkInsertTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/ConfigUtilsTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/HttpHeaderUtilsTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/JavaNetHttpClientFactoryTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/uri/CharArrayBufferTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/uri/ParserCursorTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/uri/TokenParserTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/uri/URIBuilderTest.java
create mode 100644 flink-connector-http/src/test/java/org/apache/flink/connector/http/utils/uri/URLEncodedUtilsTest.java
create mode 100644 flink-connector-http/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
create mode 100644 flink-connector-http/src/test/resources/auth/AuthResult.json
create mode 100644 flink-connector-http/src/test/resources/http-array-result-with-nulls/HttpResult.json
create mode 100644 flink-connector-http/src/test/resources/http-array-result/HttpResult.json
create mode 100644 flink-connector-http/src/test/resources/http/HttpResult.json
create mode 100644 flink-connector-http/src/test/resources/json/sink/allInOneBatch.txt
create mode 100644 flink-connector-http/src/test/resources/json/sink/fourSingleEventBatches.txt
create mode 100644 flink-connector-http/src/test/resources/json/sink/threeBatches.txt
create mode 100644 flink-connector-http/src/test/resources/json/sink/twoBatches.txt
create mode 100644 flink-connector-http/src/test/resources/security/certs/ca.crt
create mode 100644 flink-connector-http/src/test/resources/security/certs/ca.key
create mode 100644 flink-connector-http/src/test/resources/security/certs/ca.pass.key
create mode 100644 flink-connector-http/src/test/resources/security/certs/ca.srl
create mode 100644 flink-connector-http/src/test/resources/security/certs/ca_server_bundle.cert.pem
create mode 100644 flink-connector-http/src/test/resources/security/certs/client.crt
create mode 100644 flink-connector-http/src/test/resources/security/certs/client.csr
create mode 100644 flink-connector-http/src/test/resources/security/certs/client.key
create mode 100644 flink-connector-http/src/test/resources/security/certs/client.pass.key
create mode 100644 flink-connector-http/src/test/resources/security/certs/clientPrivateKey.der
create mode 100644 flink-connector-http/src/test/resources/security/certs/clientPrivateKey.pem
create mode 100644 flink-connector-http/src/test/resources/security/certs/client_keyStore.p12
create mode 100644 flink-connector-http/src/test/resources/security/certs/server.crt
create mode 100644 flink-connector-http/src/test/resources/security/certs/server.csr
create mode 100644 flink-connector-http/src/test/resources/security/certs/server.key
create mode 100644 flink-connector-http/src/test/resources/security/certs/server.pass.key
create mode 100644 flink-connector-http/src/test/resources/security/certs/serverKeyStore.jks
create mode 100644 flink-connector-http/src/test/resources/security/certs/serverTrustStore.jks
create mode 100644 flink-sql-connector-http/pom.xml
create mode 100644 flink-sql-connector-http/src/main/resources/META-INF/NOTICE
create mode 100644 flink-sql-connector-http/src/test/java/org/apache/flink/connector/http/PackagingITCase.java
create mode 100644 lombok.config
create mode 100644 pom.xml
create mode 100644 tools/ci/log4j.properties
create mode 100644 tools/maven/checkstyle.xml
create mode 100644 tools/maven/suppressions.xml
(flink-connector-http) 01/01: [FLINK-38245] Initial contribution of Getindata connector
Posted by fc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fcsaky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-http.git
commit 645c950c284e663aa67f4d888b526ba8971f2521
Author: davidradl <da...@uk.ibm.com>
AuthorDate: Thu Sep 11 09:02:19 2025 +0100
[FLINK-38245] Initial contribution of Getindata connector
Closes #1
---
.checkstyle | 15 +
.github/boring-cyborg.yml | 86 ++
.github/pull_request_template.md | 9 -
.github/workflows/build.yml | 51 -
.github/workflows/prepare_release_branch.yml | 73 --
.github/workflows/publish.yml | 53 -
.github/workflows/push_pr.yml | 37 +
.github/workflows/weekly.yml | 41 +
.gitignore | 35 +-
.mvn/jvm.config | 7 +
.mvn/wrapper/maven-wrapper.properties | 22 +
CONTRIBUTING.md | 9 -
NOTICE | 17 +
README.md | 726 +-----------
dev/README.md | 36 -
dev/checkstyle-suppressions.xml | 31 -
dev/checkstyle.xml | 218 ----
docs/JoinTable.PNG | Bin 64354 -> 0 bytes
docs/content/docs/datastream/http.md | 179 +++
docs/content/docs/table/http.md | 545 +++++++++
flink-connector-http/pom.xml | 465 ++++++++
.../connector/http/HttpPostRequestCallback.java | 46 +
.../http/HttpPostRequestCallbackFactory.java | 79 ++
.../org/apache/flink/connector/http/HttpSink.java | 90 ++
.../flink/connector}/http/HttpSinkBuilder.java | 106 +-
.../HttpStatusCodeValidationFailedException.java | 33 +
.../org/apache/flink/connector/http/LookupArg.java | 33 +
.../flink/connector/http/LookupQueryCreator.java | 43 +
.../connector/http/LookupQueryCreatorFactory.java | 68 ++
.../http/SchemaLifecycleAwareElementConverter.java | 28 +-
.../http}/auth/OidcAccessTokenManager.java | 87 +-
.../connector/http/clients/PollingClient.java | 42 +
.../http/clients/PollingClientFactory.java | 36 +
.../connector/http/clients/SinkHttpClient.java | 43 +
.../http/clients/SinkHttpClientBuilder.java | 40 +
.../http/clients/SinkHttpClientResponse.java | 42 +
.../connector/http/config/ConfigException.java | 52 +
.../http/config/HttpConnectorConfigConstants.java | 159 +++
.../http/config/SinkRequestSubmitMode.java | 34 +
.../BasicAuthHeaderValuePreprocessor.java | 38 +-
.../preprocessor/ComposeHeaderPreprocessor.java | 60 +
.../http/preprocessor/HeaderPreprocessor.java | 33 +
.../http/preprocessor/HeaderValuePreprocessor.java | 35 +
.../OIDCAuthHeaderValuePreprocessor.java | 64 +
.../connector/http/retry/HttpClientWithRetry.java | 93 ++
.../connector/http}/retry/RetryConfigProvider.java | 43 +-
.../connector/http/retry/RetryStrategyType.java | 45 +
.../connector/http}/security/SecurityContext.java | 122 +-
.../http}/security/SelfSignedTrustManager.java | 37 +-
.../connector/http/sink/HttpSinkInternal.java | 202 ++++
.../connector/http/sink/HttpSinkRequestEntry.java | 48 +
.../flink/connector/http/sink/HttpSinkWriter.java | 161 +++
.../http/sink/HttpSinkWriterStateSerializer.java | 53 +
.../sink/httpclient/AbstractRequestSubmitter.java | 64 +
.../sink/httpclient/BatchRequestSubmitter.java | 85 +-
.../httpclient/BatchRequestSubmitterFactory.java | 89 ++
.../http/sink/httpclient/HttpRequest.java | 33 +
.../httpclient/JavaNetHttpResponseWrapper.java | 46 +
.../sink/httpclient/JavaNetSinkHttpClient.java | 137 +++
.../PerRequestRequestSubmitterFactory.java | 50 +
.../http/sink/httpclient/PerRequestSubmitter.java | 94 ++
.../http/sink/httpclient/RequestSubmitter.java | 30 +
.../sink/httpclient/RequestSubmitterFactory.java | 26 +
.../http/status/ComposeHttpStatusCodeChecker.java | 176 +++
.../connector/http/status/HttpCodesParser.java | 85 ++
.../http}/status/HttpResponseChecker.java | 38 +-
.../http/status/HttpResponseCodeType.java | 63 +
.../http/status/HttpStatusCodeChecker.java | 34 +
.../status/IncludeListHttpStatusCodeChecker.java | 39 +
.../status/SingleValueHttpStatusCodeChecker.java | 44 +
.../http/status/TypeStatusCodeChecker.java | 61 +
.../table/SerializationSchemaElementConverter.java | 61 +
.../table/lookup/AsyncHttpTableLookupFunction.java | 131 +++
.../http/table/lookup/BodyBasedRequestFactory.java | 88 ++
.../http/table/lookup/GetRequestFactory.java | 90 ++
.../http/table/lookup/HttpLookupConfig.java | 48 +
.../table/lookup/HttpLookupConnectorOptions.java | 200 ++++
.../table/lookup/HttpLookupSourceRequestEntry.java | 47 +
.../http}/table/lookup/HttpLookupTableSource.java | 169 ++-
.../table/lookup/HttpLookupTableSourceFactory.java | 232 ++++
.../http/table/lookup/HttpRequestFactory.java | 35 +
.../table/lookup/HttpTableLookupFunction.java | 45 +-
.../table/lookup/JavaNetHttpPollingClient.java | 151 ++-
.../lookup/JavaNetHttpPollingClientFactory.java | 46 +
.../http/table/lookup/LookupQueryInfo.java | 86 ++
.../connector/http}/table/lookup/LookupRow.java | 43 +-
.../http/table/lookup/LookupSchemaEntry.java | 43 +
.../http}/table/lookup/RequestFactoryBase.java | 93 +-
.../table/lookup/RowDataLookupSchemaEntryBase.java | 48 +
.../RowDataSingleValueLookupSchemaEntry.java | 49 +-
.../table/lookup/RowTypeLookupSchemaEntry.java | 46 +-
.../lookup/Slf4JHttpLookupPostRequestCallback.java | 84 ++
.../Slf4jHttpLookupPostRequestCallbackFactory.java | 53 +
.../http}/table/lookup/TableSourceHelper.java | 35 +-
.../ElasticSearchLiteQueryCreator.java | 69 ++
.../ElasticSearchLiteQueryCreatorFactory.java | 56 +
.../querycreators/GenericGetQueryCreator.java | 58 +
.../GenericGetQueryCreatorFactory.java | 56 +
.../GenericJsonAndUrlQueryCreator.java | 218 ++--
.../GenericJsonAndUrlQueryCreatorFactory.java | 157 +++
.../querycreators/GenericJsonQueryCreator.java | 71 ++
.../GenericJsonQueryCreatorFactory.java | 92 ++
.../lookup/querycreators/ObjectMapperAdapter.java | 42 +
.../lookup/querycreators/PrefixedConfigOption.java | 102 ++
.../QueryFormatAwareConfiguration.java | 31 +-
.../http}/table/sink/HttpDynamicSink.java | 200 ++--
.../sink/HttpDynamicSinkConnectorOptions.java | 44 +
.../table/sink/HttpDynamicTableSinkFactory.java | 116 ++
.../table/sink/Slf4jHttpPostRequestCallback.java | 69 ++
.../sink/Slf4jHttpPostRequestCallbackFactory.java | 53 +
.../flink/connector/http}/utils/ConfigUtils.java | 85 +-
.../connector/http/utils/HttpHeaderUtils.java | 150 +++
.../http}/utils/JavaNetHttpClientFactory.java | 160 ++-
.../flink/connector/http/utils/ProxyConfig.java | 61 +
.../http/utils/SerializationSchemaUtils.java | 67 ++
.../utils/SynchronizedSerializationSchema.java | 19 +-
.../flink/connector/http/utils/ThreadUtils.java | 37 +
.../connector/http}/utils/uri/CharArrayBuffer.java | 54 +-
.../connector/http/utils/uri/NameValuePair.java | 43 +
.../connector/http/utils/uri/ParserCursor.java | 72 ++
.../connector/http}/utils/uri/TokenParser.java | 92 +-
.../connector/http}/utils/uri/URIBuilder.java | 69 +-
.../connector/http}/utils/uri/URLEncodedUtils.java | 145 +--
.../org.apache.flink.table.factories.Factory | 28 +
.../http/HttpPostRequestCallbackFactoryTest.java | 194 ++++
.../connector/http/HttpsConnectionTestBase.java | 62 +
.../flink/connector/http/StreamTableJob.java | 77 ++
.../apache/flink/connector/http/TestHelper.java | 65 ++
.../http/TestLookupPostRequestCallbackFactory.java | 53 +
.../http/TestPostRequestCallbackFactory.java | 51 +
.../http/WireMockServerPortAllocator.java | 36 +
.../flink/connector/http/app/HttpStubApp.java | 57 +
.../flink/connector/http/app/JsonTransform.java | 125 ++
.../http}/auth/OidcAccessTokenManagerTest.java | 51 +-
.../connector/http/config/ConfigExceptionTest.java | 40 +
.../http}/retry/HttpClientWithRetryTest.java | 86 +-
.../http/retry/RetryConfigProviderTest.java | 89 ++
.../http}/retry/RetryStrategyTypeTest.java | 37 +-
.../connector/http/sink/HttpSinkBuilderTest.java | 93 ++
.../http/sink/HttpSinkConnectionTest.java | 378 ++++++
.../sink/HttpSinkWriterStateSerializerTest.java | 52 +
.../connector/http}/sink/HttpSinkWriterTest.java | 90 +-
.../BatchRequestSubmitterFactoryTest.java | 96 ++
.../sink/httpclient/BatchRequestSubmitterTest.java | 103 ++
.../JavaNetSinkHttpClientConnectionTest.java | 329 ++++++
.../sink/httpclient/JavaNetSinkHttpClientTest.java | 102 +-
.../status/ComposeHttpStatusCodeCheckerTest.java | 181 +++
.../http}/status/HttpCodesParserTest.java | 79 +-
.../http}/status/HttpResponseCheckerTest.java | 55 +-
.../BasicAuthHeaderValuePreprocessorTest.java | 46 +
.../http/table/ComposeHeaderPreprocessorTest.java | 43 +
.../lookup/AsyncHttpTableLookupFunctionTest.java | 188 +--
.../table/lookup/BodyBasedRequestFactoryTest.java | 89 +-
.../lookup/HttpLookupTableSourceFactoryTest.java | 139 +++
.../lookup/HttpLookupTableSourceITCaseTest.java | 1118 ++++++++++++++++++
.../table/lookup/HttpLookupTableSourceTest.java | 258 +++++
.../JavaNetHttpPollingClientConnectionTest.java | 256 ++--
.../JavaNetHttpPollingClientFactoryTest.java | 51 +
...avaNetHttpPollingClientHttpsConnectionTest.java | 363 ++++++
.../table/lookup/JavaNetHttpPollingClientTest.java | 237 ++++
.../JavaNetHttpPollingClientWithWireTest.java | 191 +++
.../connector/http/table/lookup/JsonTransform.java | 126 ++
.../http}/table/lookup/LookupQueryInfoTest.java | 43 +-
.../RowDataSingleValueLookupSchemaEntryTest.java | 47 +
.../table/lookup/RowTypeLookupSchemaEntryTest.java | 177 +++
.../http/table/lookup/TableSourceHelperTest.java | 53 +
.../lookup/querycreators/CustomFormatFactory.java | 32 +-
.../querycreators/CustomJsonFormatFactory.java | 32 +-
.../ElasticSearchLiteQueryCreatorTest.java | 158 +++
.../querycreators/GenericGetQueryCreatorTest.java | 208 ++++
.../GenericJsonAndUrlQueryCreatorFactoryTest.java | 138 +++
.../GenericJsonAndUrlQueryCreatorTest.java | 229 ++--
.../GenericJsonQueryCreatorFactoryTest.java | 111 ++
.../querycreators/GenericJsonQueryCreatorTest.java | 49 +-
.../table/lookup/querycreators/PersonBean.java | 28 +
.../lookup/querycreators/QueryCreatorUtils.java | 54 +-
.../QueryFormatAwareConfigurationTest.java | 59 +
.../BatchRequestHttpDynamicSinkInsertTest.java | 364 ++++++
.../http/table/sink/HttpDynamicSinkTest.java | 134 +++
.../sink/HttpDynamicTableSinkFactoryTest.java | 113 ++
.../sink/PerRequestHttpDynamicSinkInsertTest.java | 331 ++++++
.../connector/http/utils/ConfigUtilsTest.java | 292 +++++
.../connector/http/utils/HttpHeaderUtilsTest.java | 47 +
.../http/utils/JavaNetHttpClientFactoryTest.java | 141 +++
.../http/utils/uri/CharArrayBufferTest.java | 96 ++
.../connector/http/utils/uri/ParserCursorTest.java | 49 +
.../connector/http/utils/uri/TokenParserTest.java | 41 +
.../connector/http}/utils/uri/URIBuilderTest.java | 183 +--
.../http}/utils/uri/URLEncodedUtilsTest.java | 57 +-
.../org.apache.flink.table.factories.Factory | 24 +
.../src}/test/resources/auth/AuthResult.json | 0
.../http-array-result-with-nulls/HttpResult.json | 0
.../resources/http-array-result/HttpResult.json | 0
.../src}/test/resources/http/HttpResult.json | 0
.../test/resources/json/sink/allInOneBatch.txt | 0
.../resources/json/sink/fourSingleEventBatches.txt | 0
.../src}/test/resources/json/sink/threeBatches.txt | 0
.../src}/test/resources/json/sink/twoBatches.txt | 0
.../src}/test/resources/security/certs/ca.crt | 0
.../src}/test/resources/security/certs/ca.key | 0
.../src}/test/resources/security/certs/ca.pass.key | 0
.../src}/test/resources/security/certs/ca.srl | 0
.../security/certs/ca_server_bundle.cert.pem | 0
.../src}/test/resources/security/certs/client.crt | 0
.../src}/test/resources/security/certs/client.csr | 0
.../src}/test/resources/security/certs/client.key | 0
.../test/resources/security/certs/client.pass.key | 0
.../resources/security/certs/clientPrivateKey.der | Bin
.../resources/security/certs/clientPrivateKey.pem | 0
.../resources/security/certs/client_keyStore.p12 | Bin
.../src}/test/resources/security/certs/server.crt | 0
.../src}/test/resources/security/certs/server.csr | 0
.../src}/test/resources/security/certs/server.key | 0
.../test/resources/security/certs/server.pass.key | 0
.../resources/security/certs/serverKeyStore.jks | Bin
.../resources/security/certs/serverTrustStore.jks | Bin
flink-sql-connector-http/pom.xml | 147 +++
.../src/main/resources/META-INF/NOTICE | 12 +
.../flink/connector/http/PackagingITCase.java | 42 +
pom.xml | 1223 ++++++++++----------
.../connectors/http/HttpPostRequestCallback.java | 29 -
.../http/HttpPostRequestCallbackFactory.java | 67 --
.../com/getindata/connectors/http/HttpSink.java | 74 --
.../HttpStatusCodeValidationFailedException.java | 15 -
.../com/getindata/connectors/http/LookupArg.java | 22 -
.../connectors/http/LookupQueryCreator.java | 27 -
.../connectors/http/LookupQueryCreatorFactory.java | 49 -
.../http/internal/ComposeHeaderPreprocessor.java | 44 -
.../http/internal/HeaderPreprocessor.java | 17 -
.../http/internal/HeaderValuePreprocessor.java | 18 -
.../internal/OIDCAuthHeaderValuePreprocessor.java | 50 -
.../connectors/http/internal/PollingClient.java | 25 -
.../http/internal/PollingClientFactory.java | 16 -
.../connectors/http/internal/SinkHttpClient.java | 28 -
.../http/internal/SinkHttpClientBuilder.java | 27 -
.../http/internal/SinkHttpClientResponse.java | 31 -
.../http/internal/config/ConfigException.java | 30 -
.../config/HttpConnectorConfigConstants.java | 143 ---
.../internal/config/SinkRequestSubmitMode.java | 17 -
.../http/internal/retry/HttpClientWithRetry.java | 71 --
.../http/internal/retry/RetryStrategyType.java | 27 -
.../http/internal/sink/HttpSinkInternal.java | 188 ---
.../http/internal/sink/HttpSinkRequestEntry.java | 36 -
.../http/internal/sink/HttpSinkWriter.java | 131 ---
.../sink/HttpSinkWriterStateSerializer.java | 36 -
.../sink/httpclient/AbstractRequestSubmitter.java | 49 -
.../httpclient/BatchRequestSubmitterFactory.java | 76 --
.../http/internal/sink/httpclient/HttpRequest.java | 16 -
.../httpclient/JavaNetHttpResponseWrapper.java | 33 -
.../sink/httpclient/JavaNetSinkHttpClient.java | 121 --
.../PerRequestRequestSubmitterFactory.java | 33 -
.../sink/httpclient/PerRequestSubmitter.java | 80 --
.../internal/sink/httpclient/RequestSubmitter.java | 16 -
.../sink/httpclient/RequestSubmitterFactory.java | 8 -
.../status/ComposeHttpStatusCodeChecker.java | 161 ---
.../http/internal/status/HttpCodesParser.java | 63 -
.../http/internal/status/HttpResponseCodeType.java | 49 -
.../internal/status/HttpStatusCodeChecker.java | 16 -
.../status/SingleValueHttpStatusCodeChecker.java | 28 -
.../internal/status/TypeStatusCodeChecker.java | 43 -
.../status/WhiteListHttpStatusCodeChecker.java | 23 -
.../table/SerializationSchemaElementConverter.java | 47 -
.../table/lookup/AsyncHttpTableLookupFunction.java | 111 --
.../table/lookup/BodyBasedRequestFactory.java | 71 --
.../internal/table/lookup/GetRequestFactory.java | 70 --
.../internal/table/lookup/HttpLookupConfig.java | 33 -
.../table/lookup/HttpLookupConnectorOptions.java | 161 ---
.../table/lookup/HttpLookupSourceRequestEntry.java | 32 -
.../table/lookup/HttpLookupTableSourceFactory.java | 194 ----
.../internal/table/lookup/HttpRequestFactory.java | 20 -
.../lookup/JavaNetHttpPollingClientFactory.java | 34 -
.../internal/table/lookup/LookupQueryInfo.java | 73 --
.../internal/table/lookup/LookupSchemaEntry.java | 28 -
.../table/lookup/RowDataLookupSchemaEntryBase.java | 36 -
.../lookup/Slf4JHttpLookupPostRequestCallback.java | 70 --
.../Slf4jHttpLookupPostRequestCallbackFactory.java | 38 -
.../ElasticSearchLiteQueryCreator.java | 50 -
.../ElasticSearchLiteQueryCreatorFactory.java | 44 -
.../querycreators/GenericGetQueryCreator.java | 42 -
.../GenericGetQueryCreatorFactory.java | 43 -
.../GenericJsonAndUrlQueryCreatorFactory.java | 142 ---
.../querycreators/GenericJsonQueryCreator.java | 59 -
.../GenericJsonQueryCreatorFactory.java | 78 --
.../lookup/querycreators/ObjectMapperAdapter.java | 28 -
.../lookup/querycreators/PrefixedConfigOption.java | 85 --
.../sink/HttpDynamicSinkConnectorOptions.java | 27 -
.../table/sink/HttpDynamicTableSinkFactory.java | 100 --
.../table/sink/Slf4jHttpPostRequestCallback.java | 53 -
.../sink/Slf4jHttpPostRequestCallbackFactory.java | 39 -
.../http/internal/utils/ExceptionUtils.java | 26 -
.../http/internal/utils/HttpHeaderUtils.java | 137 ---
.../http/internal/utils/ProxyConfig.java | 40 -
.../internal/utils/SerializationSchemaUtils.java | 50 -
.../http/internal/utils/ThreadUtils.java | 17 -
.../http/internal/utils/uri/NameValuePair.java | 56 -
.../http/internal/utils/uri/ParserCursor.java | 92 --
.../org.apache.flink.table.factories.Factory | 8 -
src/test/java/com/getindata/StreamTableJob.java | 56 -
.../connectors/http/ExceptionUtilsTest.java | 18 -
.../http/HttpPostRequestCallbackFactoryTest.java | 172 ---
.../com/getindata/connectors/http/TestHelper.java | 48 -
.../http/TestLookupPostRequestCallbackFactory.java | 29 -
.../http/TestPostRequestCallbackFactory.java | 27 -
.../getindata/connectors/http/app/HttpStubApp.java | 35 -
.../connectors/http/app/JsonTransform.java | 108 --
.../BasicAuthHeaderValuePreprocessorTest.java | 26 -
.../internal/ComposeHeaderPreprocessorTest.java | 21 -
.../http/internal/HttpsConnectionTestBase.java | 47 -
.../http/internal/config/ConfigExceptionTest.java | 20 -
.../internal/retry/RetryConfigProviderTest.java | 63 -
.../http/internal/sink/HttpSinkBuilderTest.java | 74 --
.../http/internal/sink/HttpSinkConnectionTest.java | 325 ------
.../sink/HttpSinkWriterStateSerializerTest.java | 30 -
.../BatchRequestSubmitterFactoryTest.java | 86 --
.../sink/httpclient/BatchRequestSubmitterTest.java | 96 --
.../JavaNetSinkHttpClientConnectionTest.java | 369 ------
.../status/ComposeHttpStatusCodeCheckerTest.java | 171 ---
.../lookup/HttpLookupTableSourceFactoryTest.java | 110 --
.../lookup/HttpLookupTableSourceITCaseTest.java | 1071 -----------------
.../table/lookup/HttpLookupTableSourceTest.java | 253 ----
.../JavaNetHttpPollingClientFactoryTest.java | 30 -
...avaNetHttpPollingClientHttpsConnectionTest.java | 356 ------
.../table/lookup/JavaNetHttpPollingClientTest.java | 227 ----
.../JavaNetHttpPollingClientWithWireTest.java | 163 ---
.../http/internal/table/lookup/JsonTransform.java | 108 --
.../RowDataSingleValueLookupSchemaEntryTest.java | 28 -
.../table/lookup/RowTypeLookupSchemaEntryTest.java | 144 ---
.../table/lookup/TableSourceHelperTest.java | 34 -
.../ElasticSearchLiteQueryCreatorTest.java | 139 ---
.../querycreators/GenericGetQueryCreatorTest.java | 181 ---
.../GenericJsonAndUrlQueryCreatorFactoryTest.java | 117 --
.../GenericJsonQueryCreatorFactoryTest.java | 88 --
.../table/lookup/querycreators/PathBean.java | 11 -
.../table/lookup/querycreators/PersonBean.java | 12 -
.../QueryFormatAwareConfigurationTest.java | 38 -
.../BatchRequestHttpDynamicSinkInsertTest.java | 341 ------
.../internal/table/sink/HttpDynamicSinkTest.java | 109 --
.../sink/HttpDynamicTableSinkFactoryTest.java | 94 --
.../sink/PerRequestHttpDynamicSinkInsertTest.java | 309 -----
.../http/internal/utils/ConfigUtilsTest.java | 244 ----
.../http/internal/utils/HttpHeaderUtilsTest.java | 27 -
.../utils/JavaNetHttpClientFactoryTest.java | 112 --
.../internal/utils/uri/CharArrayBufferTest.java | 73 --
.../http/internal/utils/uri/ParserCursorTest.java | 29 -
.../http/internal/utils/uri/TokenParserTest.java | 25 -
.../org.apache.flink.table.factories.Factory | 4 -
tools/ci/log4j.properties | 72 ++
tools/maven/checkstyle.xml | 564 +++++++++
tools/maven/suppressions.xml | 27 +
349 files changed, 17978 insertions(+), 14033 deletions(-)
diff --git a/.checkstyle b/.checkstyle
new file mode 100644
index 0000000..8a4c73c
--- /dev/null
+++ b/.checkstyle
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<fileset-config file-format-version="1.2.0" simple-config="false" sync-formatter="false">
+ <local-check-config name="maven-checkstyle-plugin validate" location="file:/Users/davidradley/flink-http-connector/tools/maven/checkstyle.xml" type="remote" description="maven-checkstyle-plugin configuration validate">
+ <property name="checkstyle.header.file" value="/Users/davidradley/eclipse-http-flink/.metadata/.plugins/org.eclipse.core.resources/.projects/flink-connector-http/com.basistech.m2e.code.quality.checkstyleConfigurator/checkstyle-header-validate.txt"/>
+ <property name="checkstyle.cache.file" value="${project_loc}/target/checkstyle-cachefile"/>
+ <property name="checkstyle.suppressions.file" value="/Users/davidradley/eclipse-http-flink/.metadata/.plugins/org.eclipse.core.resources/.projects/flink-connector-http/com.basistech.m2e.code.quality.checkstyleConfigurator/checkstyle-suppressions-validate.xml"/>
+ </local-check-config>
+ <fileset name="java-sources-validate" enabled="true" check-config-name="maven-checkstyle-plugin validate" local="true">
+ <file-match-pattern match-pattern="^src/main/java/.*\/.*\.java" include-pattern="true"/>
+ <file-match-pattern match-pattern="^src/test/java/.*\/.*\.java" include-pattern="true"/>
+ <file-match-pattern match-pattern="^src/main/resources/.*\.properties" include-pattern="true"/>
+ <file-match-pattern match-pattern="^src/test/resources/.*\.properties" include-pattern="true"/>
+ </fileset>
+</fileset-config>
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
new file mode 100644
index 0000000..f8bc106
--- /dev/null
+++ b/.github/boring-cyborg.yml
@@ -0,0 +1,86 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+labelPRBasedOnFilePath:
+ component=BuildSystem:
+ - .github/**/*
+ - tools/maven/*
+
+ component=Documentation:
+ - docs/**/*
+
+ component=Connectors/Http:
+ - flink-connector-http*/**/*
+
+###### IssueLink Adder #################################################################################################
+# Insert Issue (Jira/Github etc) link in PR description based on the Issue ID in PR title.
+insertIssueLinkInPrDescription:
+ # specify the placeholder for the issue link that should be present in the description
+ descriptionIssuePlaceholderRegexp: "^Issue link: (.*)$"
+ matchers:
+ # you can have several matches - for different types of issues
+ # only the first matching entry is replaced
+ jiraIssueMatch:
+ # specify the regexp of issue id that you can find in the title of the PR
+ # the match groups can be used to build the issue id (${1}, ${2}, etc.).
+ titleIssueIdRegexp: \[(FLINK-[0-9]+)\]
+ # the issue link to be added. ${1}, ${2} ... are replaced with the match groups from the
+ # title match (remember to use quotes)
+ descriptionIssueLink: "[${1}](https://issues.apache.org/jira/browse/${1}/)"
+ docOnlyIssueMatch:
+ titleIssueIdRegexp: \[hotfix\]
+ descriptionIssueLink: "`Documentation only change, no JIRA issue`"
+
+###### Title Validator #################################################################################################
+# Verifies if commit/PR titles match the regexp specified
+verifyTitles:
+ # Regular expression that should be matched by titles of commits or PR
+ titleRegexp: ^\[FLINK-[0-9]+\].*$|^\[FLINK-XXXXX\].*$|^\[hotfix].*$
+ # If set to true, it will always check the PR title (as opposed to the individual commits).
+ alwaysUsePrTitle: false
+ # If set to true, it will only check the commit in case there is a single commit.
+ # In case of multiple commits it will check PR title.
+ # This reflects the standard behaviour of Github that for `Squash & Merge` GitHub
+ # uses the PR title rather than commit messages for the squashed commit ¯\_(ツ)_/¯
+ # For single-commit PRs it takes the squashed commit message from the commit as expected.
+ #
+ # If set to false it will check all commit messages. This is useful when you do not squash commits at merge.
+ validateEitherPrOrSingleCommitTitle: true
+ # The title the GitHub status should appear from.
+ statusTitle: "Title Validator"
+ # A custom message to be displayed when the title passes validation.
+ successMessage: "Validation successful!"
+ # A custom message to be displayed when the title fails validation.
+ # Allows insertion of ${type} (commit/PR), ${title} (the title validated) and ${regex} (the titleRegexp above).
+ failureMessage: "Wrong ${type} title: ${title}"
+
+# Various Flags to control behaviour of the "Labeler"
+labelerFlags:
+ # If this flag is changed to 'false', labels would only be added when the PR is first created
+ # and not when existing PR is updated.
+ # The default is 'true' which means the labels would be added when PR is updated even if they
+ # were removed by the user
+ labelOnPRUpdates: true
+
+# Comment to be posted to welcome users when they open their first PR
+firstPRWelcomeComment: >
+ Thanks for opening this pull request! Please check out our contributing guidelines. (https://flink.apache.org/contributing/how-to-contribute.html)
+
+# Comment to be posted to congratulate user on their first merged PR
+firstPRMergeComment: >
+ Awesome work, congrats on your first merged pull request!
\ No newline at end of file
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
deleted file mode 100644
index 44b483a..0000000
--- a/.github/pull_request_template.md
+++ /dev/null
@@ -1,9 +0,0 @@
-#### Description
-
-`describe the purpose of the change here`
-
-Resolves `<issue nr here>`
-
-##### PR Checklist
-- [ ] Tests added
-- [ ] [Changelog](CHANGELOG.md) updated
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
deleted file mode 100644
index b20dd4d..0000000
--- a/.github/workflows/build.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-name: Build
-
-on:
- push:
- branches:
- - 'main'
- - 'release/**'
- pull_request:
-
-env:
- MAVEN_CLI_OPTS: "--batch-mode"
- MAVEN_OPTS: "-Dmaven.repo.local=.m2/repository"
- JAVA_ADDITIONAL_OPTS: "-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss,SSS"
- FF_USE_FASTZIP: "true"
-
-jobs:
- build:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- flink: ["1.18.1", "1.19.1", "1.20.0"]
- steps:
- - uses: actions/checkout@v3
-
- - name: Set up JDK 11
- uses: actions/setup-java@v3
- with:
- java-version: '11'
- distribution: 'adopt'
- cache: maven
-
- - name: Build for Flink ${{ matrix.flink }}
- run: mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS -Dflink.version=${{ matrix.flink }} compile
-
- - name: Tests for Flink ${{ matrix.flink }}
- run: |
- mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS -Dflink.version=${{ matrix.flink }} test integration-test
-
- - name: Test JavaDoc
- run: mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS javadoc:javadoc
- if: startsWith(matrix.flink, '1.20')
-
- - name: Add coverage to PR
- id: jacoco
- uses: madrapps/jacoco-report@v1.7.1
- with:
- paths: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
- token: ${{ secrets.GITHUB_TOKEN }}
- min-coverage-overall: 40
- min-coverage-changed-files: 60
- if: startsWith(matrix.flink, '1.20') && github.event.pull_request.head.repo.fork == false
diff --git a/.github/workflows/prepare_release_branch.yml b/.github/workflows/prepare_release_branch.yml
deleted file mode 100644
index ff2993f..0000000
--- a/.github/workflows/prepare_release_branch.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-name: Prepare release branch
-
-on:
- workflow_dispatch:
- inputs:
- version_part:
- description: The part of the version to update (Patch, Minor or Major)
- required: true
- type: choice
- options:
- - Patch
- - Minor
- - Major
- default: 'Minor'
-
-jobs:
- prepare-branch:
- runs-on: ubuntu-latest
-
- permissions: # Permissions to create a new branch and a new PR.
- contents: write
- pull-requests: write
-
- steps:
- - uses: actions/checkout@v3
-
- - name: Set up JDK 11
- uses: actions/setup-java@v3
- with:
- java-version: '11'
- distribution: 'adopt'
- cache: maven
-
- - name: Validate inputs
- run: |
- echo "INPUT_VERSION_PART: ${{ github.event.inputs.version_part }}"
- python -c "if '${{ github.event.inputs.version_part }}' not in ['Patch', 'Minor', 'Major']: raise ValueError(\"'${{ github.event.inputs.version_part }}' must be one of ['Patch', 'Minor', 'Major'])\")"
-
- - name: Save current version
- id: save_current_version
- run: |
- mvn versions:set -DremoveSnapshot -DgenerateBackupPoms=false
- echo "::set-output name=current_version::$(mvn -B help:evaluate -Dexpression=project.version -q -DforceStdout)"
-
- - name: Update the CHANGELOG according to 'Keep a Changelog' guidelines
- uses: thomaseizinger/keep-a-changelog-new-release@v1
- with:
- version: ${{ steps.save_current_version.outputs.current_version }}
-
- - name: Create a new release branch
- run: |
- git config user.name github-actions
- git config user.email github-actions@github.com
- git commit -am "Bump CHANGELOG for release ${{ steps.save_current_version.outputs.current_version }}"
- git checkout -b release/${{ steps.save_current_version.outputs.current_version }}
- git push -u origin release/${{ steps.save_current_version.outputs.current_version }}
-
- - name: Bump development version
- run: |
- git checkout -b bump-version-after-${{ steps.save_current_version.outputs.current_version }}
- mvn validate -D 'bump${{ github.event.inputs.version_part }}' -DgenerateBackupPoms=false
- git commit -am "Bump development version to $(mvn -B help:evaluate -Dexpression=project.version -q -DforceStdout)"
- git push -u origin bump-version-after-${{ steps.save_current_version.outputs.current_version }}
-
- - name: Open a PR to bump development version to main
- id: open_pr
- uses: vsoch/pull-request-action@1.1.0
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- PULL_REQUEST_BRANCH: main
- PULL_REQUEST_FROM_BRANCH: bump-version-after-${{ steps.save_current_version.outputs.current_version }}
- PULL_REQUEST_TITLE: "Bump development version after release of ${{ steps.save_current_version.outputs.current_version }}"
- PULL_REQUEST_BODY: "Bump SNAPSHOT version and CHANGELOG for subsequent development."
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
deleted file mode 100644
index a994f28..0000000
--- a/.github/workflows/publish.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-name: Publish
-
-on:
- release:
- types: [published]
-
-env:
- MAVEN_CLI_OPTS: "--batch-mode"
- MAVEN_OPTS: "-Dmaven.repo.local=.m2/repository"
- JAVA_ADDITIONAL_OPTS: "-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss,SSS"
- FF_USE_FASTZIP: "true"
-
-jobs:
- build:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Check release tag match # ... and fail fast if they do not
- run: diff <(echo "${{ github.ref_name }}") <(echo "$(mvn -B help:evaluate -Dexpression=project.version -q -DforceStdout)")
-
- - name: Set up JDK 11
- uses: actions/setup-java@v4
- with:
- java-version: '11'
- distribution: 'temurin'
- cache: maven
-
- - name: Build
- run: mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS compile
-
- - name: Tests
- run: |
- mvn $MAVEN_CLI_OPTS $JAVA_ADDITIONAL_OPTS test integration-test
-
- - name: Set up Apache Maven Central
- uses: actions/setup-java@v4
- with:
- java-version: '11'
- distribution: 'temurin'
- server-id: ossrh
- server-username: MAVEN_USERNAME
- server-password: MAVEN_PASSWORD
- gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
- gpg-passphrase: GPG_PRIVATE_KEY_PASSWORD
-
- - name: Publish to Apache Maven Central
- if: github.event.release
- run: mvn deploy -P release
- env:
- MAVEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USERNAME }}
- MAVEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN_PASSWORD }}
- GPG_PRIVATE_KEY_PASSWORD: ${{ secrets.GPG_PRIVATE_KEY_PASSWORD }}
diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml
new file mode 100644
index 0000000..5a21114
--- /dev/null
+++ b/.github/workflows/push_pr.yml
@@ -0,0 +1,37 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# We need to specify repo related information here since Apache INFRA doesn't differentiate
+# between several workflows with the same names while preparing a report for GHA usage
+# https://infra-reports.apache.org/#ghactions
+name: Flink Connector HTTP CI
+on: [push, pull_request]
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+jobs:
+ compile_and_test:
+ strategy:
+ matrix:
+ flink: [ 1.20.0 ]
+ jdk: [ '11', '17', '21' ]
+ uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
+ with:
+ flink_version: ${{ matrix.flink }}
+ jdk_version: ${{ matrix.jdk }}
+
diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
new file mode 100644
index 0000000..c2e7d40
--- /dev/null
+++ b/.github/workflows/weekly.yml
@@ -0,0 +1,41 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# We need to specify repo related information here since Apache INFRA doesn't differentiate
+# between several workflows with the same names while preparing a report for GHA usage
+# https://infra-reports.apache.org/#ghactions
+name: Weekly Flink Connector HTTP CI
+on:
+ schedule:
+ - cron: "0 0 * * 0"
+ workflow_dispatch:
+jobs:
+ compile_and_test:
+ if: github.repository_owner == 'apache'
+ strategy:
+ matrix:
+ flink_branches: [{
+ flink: 1.20-0,
+ branch: main
+ }]
+ uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils
+ with:
+ flink_version: ${{ matrix.flink_branches.flink }}
+ connector_branch: ${{ matrix.flink_branches.branch }}
+ jdk_version: ${{ matrix.flink_branches.jdk || '11, 17, 21' }}
+ run_dependency_convergence: false
diff --git a/.gitignore b/.gitignore
index 637d596..bbad081 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,12 +5,33 @@
.gitignore.swp
.project
.settings
+/.java-version
+.eslintcache
+.cache
+scalastyle-output.xml
+.idea/*
+./bobmodes/*
+!.idea/vcs.xml
+.metadata
+.version.properties
+filter.properties
+logs.zip
+.mvn/wrapper/*.jar
+.mvn/maven.config
+target
+tmp
target
bin
-/flink.http.connector.iml
-/src/main/flink-http-connector.iml
-/src/main/main.iml
-/src/test/test.iml
-/flink-http-connector.iml
-/dependency-reduced-pom.xml
-/.java-version
+*.class
+*.iml
+*.swp
+*.jar
+*.zip
+*.log
+*.pyc
+.DS_Store
+build-target
+atlassian-ide-plugin.xml
+out/
+/docs/api
+tools/japicmp-output
diff --git a/.mvn/jvm.config b/.mvn/jvm.config
new file mode 100644
index 0000000..7a099aa
--- /dev/null
+++ b/.mvn/jvm.config
@@ -0,0 +1,7 @@
+-XX:+IgnoreUnrecognizedVMOptions
+--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
new file mode 100644
index 0000000..0fe40ae
--- /dev/null
+++ b/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# updating the Maven version requires updates to certain documentation and verification logic
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip
+distributionSha256Sum=ccf20a80e75a17ffc34d47c5c95c98c39d426ca17d670f09cd91e877072a9309
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.2/maven-wrapper-3.3.2.jar
+wrapperSha256Sum=3d8f20ce6103913be8b52aef6d994e0c54705fb527324ceb9b835b338739c7a8
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index 4c0b7d8..0000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1,9 +0,0 @@
-## PR Guidelines
-1. Fork branch from `main`.
-1. Ensure to provide unit tests for new functionality.
-1. Update documentation accordingly.
-1. Update [changelog](CHANGELOG.md) according to ["Keep a changelog"](https://keepachangelog.com/en/1.0.0/) guidelines.
-1. Squash changes with a single commit as much as possible and ensure verbose PR name.
-1. Open a PR against `main`
-
-*We reserve the right to take over and modify or abandon PRs that do not match the workflow or are abandoned.*
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..bba5e33
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,17 @@
+Apache Flink HTTP Connector
+Copyright 2025 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby
+granted, provided that this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING
+ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE
+USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+
diff --git a/README.md b/README.md
index 211eff4..72e900f 100644
--- a/README.md
+++ b/README.md
@@ -1,714 +1,68 @@
-# flink-http-connector
+# Apache Flink HTTP Connector
-[](https://mvnrepository.com/artifact/com.getindata/flink-http-connector)
-[](https://javadoc.io/doc/com.getindata/flink-http-connector)
+This repository contains the official Apache Flink HTTP connector.
-The HTTP TableLookup connector that allows for pulling data from external system via HTTP GET method and HTTP Sink that allows for sending data to external system via HTTP requests.
+## Apache Flink
-**Note**: The `main` branch may be in an *unstable or even broken state* during development.
-Please use [releases](https://github.com/getindata/flink-http-connector/releases) instead of the `main` branch in order to get a stable set of binaries.
+Apache Flink is an open source stream processing framework with powerful stream- and batch-processing capabilities.
-The goal for HTTP TableLookup connector was to use it in Flink SQL statement as a standard table that can be later joined with other stream using pure SQL Flink.
-
-Currently, HTTP source connector supports only Lookup Joins (TableLookup) [1] in Table/SQL API.
-`HttpSink` supports both Streaming API (when using [HttpSink](src/main/java/com/getindata/connectors/http/internal/sink/HttpSink.java) built using [HttpSinkBuilder](src/main/java/com/getindata/connectors/http/internal/sink/HttpSinkBuilder.java)) and the Table API (using connector created in [HttpDynamicTableSinkFactory](src/main/java/com/getindata/connectors/http/internal/table/HttpDynamicTableSinkFactory.java)).
+Learn more about Flink at [https://flink.apache.org/](https://flink.apache.org/)
-## Updating the connector
-In case of updating http-connector please see [Breaking changes](#breaking-changes) section.
+## Building the Apache Flink HTTP Connector from Source
-## Prerequisites
-* Java 11
-* Maven 3
-* Flink 1.18+. Recommended Flink 1.20.*
-
-
-
-## Runtime dependencies
-This connector has few Flink's runtime dependencies, that are expected to be provided.
-* `org.apache.flink.flink-java`
-* `org.apache.flink.flink-clients`
-* `org.apache.flink.flink-connector-base`
-
-## Installation
-
-In order to use the `flink-http-connector` the following dependencies are required for both projects using a build automation tool (such as Maven or SBT) and SQL Client with SQL JAR bundles. For build automation tool reference, look into Maven Central: [https://mvnrepository.com/artifact/com.getindata/flink-http-connector](https://mvnrepository.com/artifact/com.getindata/flink-http-connector).
-
-## Documentation
-
-You can read the official JavaDoc documentation of the latest release at [https://javadoc.io/doc/com.getindata/flink-http-connector](https://javadoc.io/doc/com.getindata/flink-http-connector).
-
-## Usage
-
-### HTTP TableLookup Source
-Flink SQL table definition:
-
-Enrichment Lookup Table
-```roomsql
-CREATE TABLE Customers (
- id STRING,
- id2 STRING,
- msg STRING,
- uuid STRING,
- details ROW<
- isActive BOOLEAN,
- nestedDetails ROW<
- balance STRING
- >
- >
-) WITH (
-'connector' = 'rest-lookup',
-'format' = 'json',
-'url' = 'http://localhost:8080/client',
-'asyncPolling' = 'true'
-)
-```
-
-Data Source Table
-```roomsql
-CREATE TABLE Orders (
- id STRING,
- id2 STRING,
- proc_time AS PROCTIME()
-) WITH (
-'connector' = 'datagen',
-'rows-per-second' = '1',
-'fields.id.kind' = 'sequence',
-'fields.id.start' = '1',
-'fields.id.end' = '120',
-'fields.id2.kind' = 'sequence',
-'fields.id2.start' = '2',
-'fields.id2.end' = '120'
-);
-```
-
-Using _Customers_ table in Flink SQL Lookup Join with _Orders_ table:
-
-```roomsql
-SELECT o.id, o.id2, c.msg, c.uuid, c.isActive, c.balance FROM Orders AS o
-JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c ON o.id = c.id AND o.id2 = c.id2
-```
-
-The columns and their values used for JOIN `ON` condition will be used as HTTP GET parameters where the column name will be used as a request parameter name.
-
-For Example:
-``
-http://localhost:8080/client/service?id=1&uuid=2
-``
-
-Or for REST POST method they will be converted to Json and used as request body. In this case, json request body will look like this:
-```json
-{
- "id": "1",
- "uuid": "2"
-}
-```
-
-#### Http headers
-It is possible to set HTTP headers that will be added to HTTP request send by lookup source connector.
-Headers are defined via property key `gid.connector.http.source.lookup.header.HEADER_NAME = header value` for example:
-`gid.connector.http.source.lookup.header.X-Content-Type-Options = nosniff`.
-
-Headers can be set using http lookup source table DDL. In example below, HTTP request done for `http-lookup` table will contain three headers:
-- `Origin`
-- `X-Content-Type-Options`
-- `Content-Type`
-
-```roomsql
-CREATE TABLE http-lookup (
- id bigint,
- some_field string
-) WITH (
- 'connector' = 'rest-lookup',
- 'format' = 'json',
- 'url' = 'http://localhost:8080/client',
- 'asyncPolling' = 'true',
- 'gid.connector.http.source.lookup.header.Origin' = '*',
- 'gid.connector.http.source.lookup.header.X-Content-Type-Options' = 'nosniff',
- 'gid.connector.http.source.lookup.header.Content-Type' = 'application/json'
-)
-```
-
-#### Custom REST query
-Http Lookup Source builds queries out of `JOIN` clauses. One can customize how those queries are built by implementing
-[LookupQueryCreator](src/main/java/com/getindata/connectors/http/LookupQueryCreator.java) and
-[LookupQueryCreatorFactory](src/main/java/com/getindata/connectors/http/LookupQueryCreatorFactory.java) interfaces.
-Custom implementations of `LookupQueryCreatorFactory` can be registered along other factories in
-`resources/META-INF.services/org.apache.flink.table.factories.Factory` file and then referenced by their identifiers in
-the Http Lookup Source DDL property field `gid.connector.http.source.lookup.query-creator`.
-
-A default implementation that builds an "ordinary" GET query, i.e. adds `?joinColumn1=value1&joinColumn2=value2&...`
-to the URI of the endpoint,
-
-For body based queries such as POST/PUT requests, the
-([GenericGetQueryCreator](src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericGetQueryCreator.java))
-is provided as a default query creator. This implementation uses Flink's [json-format](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/table/formats/json/) to convert RowData object into Json String.
-
-The `GenericGetQueryCreator` allows for using custom formats that will perform serialization to Json. Thanks to this, users can create their own logic for converting RowData to Json Strings suitable for their HTTP endpoints and use this logic as custom format
-with HTTP Lookup connector and SQL queries.
-To create a custom format user has to implement Flink's `SerializationSchema` and `SerializationFormatFactory` interfaces and register custom format factory along other factories in
-`resources/META-INF.services/org.apache.flink.table.factories.Factory` file. This is common Flink mechanism for providing custom implementations for various factories.
-
-The most flexible query creator is the [GenericJsonAndUrlQueryCreator](src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java)
-which allows column content to be mapped as URL, path, body and query parameter request values; it supports
-POST, PUT and GET operations. This query creator allows you to issue json requests without needing to code
-your own custom http connector. The mappings from columns to the json request are supplied in the query creator configuration
-parameters `gid.connector.http.request.query-param-fields`, `gid.connector.http.request.body-fields` and `gid.connector.http.request.url-map`.
-
-In order to use custom format, user has to specify option `'lookup-request.format' = 'customFormatName'`, where `customFormatName` is the identifier of custom format factory.
-
-Additionally, it is possible to pass query format options from table's DDL.
-This can be done by using option like so: `'lookup-request.format.customFormatName.customFormatProperty' = 'propertyValue'`, for example
-`'lookup-request.format.customFormatName.fail-on-missing-field' = 'true'`.
-
-It is important that `customFormatName` part match `SerializationFormatFactory` identifier used for custom format implementation.
-In this case, the `fail-on-missing-field` will be passed to `SerializationFormatFactory::createEncodingFormat(
-DynamicTableFactory.Context context, ReadableConfig formatOptions)` method in `ReadableConfig` object.
-
-With default configuration, Flink-Json format is used for `GenericGetQueryCreator`, all options defined in [json-format](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/table/formats/json/)
-can be passed through table DDL. For example `'lookup-request.format.json.fail-on-missing-field' = 'true'`. In this case, format identifier is `json`.
-
-#### Timeouts
-Lookup Source is guarded by two timeout timers. First one is specified by Flink's AsyncIO operator that executes `AsyncTableFunction`.
-The default value of this timer is set to 3 minutes and can be changed via `table.exec.async-lookup.timeout` [option](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/config/#table-exec-async-lookup-timeout).
-
-The second one is set per individual HTTP requests by HTTP client. Its default value is set currently to 30 seconds and can be changed via `gid.connector.http.source.lookup.request.timeout` option.
-
-Flink's current implementation of `AsyncTableFunction` does not allow specifying custom logic for handling Flink AsyncIO timeouts as it is for Java API.
-Because of that, if AsyncIO timer passes, Flink will throw TimeoutException which will cause job restart.
-
-#### Retries (Lookup source)
-Lookup source handles auto-retries for two scenarios:
-1. IOException occurs (e.g. temporary network outage)
-2. The response contains a HTTP error code that indicates a retriable error. These codes are defined in the table configuration (see `gid.connector.http.source.lookup.retry-codes`).
-Retries are executed silently, without restarting the job. After reaching max retries attempts (per request) operation will fail and restart job.
-
-Notice that HTTP codes are categorized into into 3 groups:
-- successful responses - response is returned immediately for further processing
-- temporary errors - request will be retried up to the retry limit
-- error responses - unexpected responses are not retried and will fail the job. Any HTTP error code which is not configured as successful or temporary error is treated as an unretriable error.
-
-##### Retry strategy
-User can choose retry strategy type for source table:
-- fixed-delay - http request will be re-sent after specified delay.
-- exponential-delay - request will be re-sent with exponential backoff strategy, limited by `lookup.max-retries` attempts. The delay for each retry is calculated as the previous attempt's delay multiplied by the backoff multiplier (parameter `gid.connector.http.source.lookup.retry-strategy.exponential-delay.backoff-multiplier`) up to `gid.connector.http.source.lookup.retry-strategy.exponential-delay.max-backoff`. The initial delay value is defined in the table configuration as `gid.conne [...]
-
-
-#### Lookup multiple results
-
-Typically, join can return zero, one or more results. What is more, there are lots of possible REST API designs and
-pagination methods. Currently, the connector supports only two simple approaches (`gid.connector.http.source.lookup.result-type`):
-
-- `single-value` - REST API returns single object.
-- `array` - REST API returns array of objects. Pagination is not supported yet.
-
-Please be informed that the mechanism will be enhanced in the future. See [HTTP-118](https://github.com/getindata/flink-http-connector/issues/118).
-
-### HTTP Sink
-The following example shows the minimum Table API example to create a [HttpDynamicSink](src/main/java/com/getindata/connectors/http/internal/table/HttpDynamicSink.java) that writes JSON values to an HTTP endpoint using POST method, assuming Flink has JAR of [JSON serializer](https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/connectors/table/formats/json/) installed:
-
-```roomsql
-CREATE TABLE http (
- id bigint,
- some_field string
-) WITH (
- 'connector' = 'http-sink',
- 'url' = 'http://example.com/myendpoint',
- 'format' = 'json'
-)
-```
-
-Then use `INSERT` SQL statement to send data to your HTTP endpoint:
-
-```roomsql
-INSERT INTO http VALUES (1, 'Ninette'), (2, 'Hedy')
-```
-
-Due to the fact that `HttpSink` sends bytes inside HTTP request's body, one can easily swap `'format' = 'json'` for some other [format](https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/connectors/table/formats/overview/).
-
-Other examples of usage of the Table API can be found in [some tests](src/test/java/com/getindata/connectors/http/table/HttpDynamicSinkInsertTest.java).
-
-### Request submission
-Starting from version 0.10 HTTP Sink by default submits events in batch. Before version 0.10 the default and only submission type was `single`.
-This is a breaking compatibility change.
-
-The submission mode can be changed using `gid.connector.http.sink.writer.request.mode` property using `single` or `batch` as property value.
-
-#### Batch submission mode
-In batch mode, a number of events (processed elements) will be batched and submitted in one HTTP request.
-In this mode, HTTP PUT/POST request's body contains a Json array, where every element of this array represents
-individual event.
-
-An example of Http Sink batch request body containing data for three events:
-```json
-[
- {
- "id": 1,
- "first_name": "Ninette",
- "last_name": "Clee",
- "gender": "Female",
- "stock": "CDZI",
- "currency": "RUB",
- "tx_date": "2021-08-24 15:22:59"
- },
- {
- "id": 2,
- "first_name": "Rob",
- "last_name": "Zombie",
- "gender": "Male",
- "stock": "DGICA",
- "currency": "GBP",
- "tx_date": "2021-10-25 20:53:54"
- },
- {
- "id": 3,
- "first_name": "Adam",
- "last_name": "Jones",
- "gender": "Male",
- "stock": "DGICA",
- "currency": "PLN",
- "tx_date": "2021-10-26 20:53:54"
- }
-]
-```
-
-By default, batch size is set to 500 which is the same as Http Sink's `maxBatchSize` property and has value of 500.
-The `maxBatchSize' property sets maximal number of events that will by buffered by Flink runtime before passing it to Http Sink for processing.
+Prerequisites:
-In order to change submission batch size use `gid.connector.http.sink.request.batch.size` property. For example:
-
-Streaming API:
-```java
-HttpSink.<String>builder()
- .setEndpointUrl("http://example.com/myendpoint")
- .setElementConverter(
- (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
- .setProperty("gid.connector.http.sink.request.batch.size", "50")
- .build();
-```
-SQL:
-```roomsql
-CREATE TABLE http (
- id bigint,
- some_field string
-) WITH (
- 'connector' = 'http-sink',
- 'url' = 'http://example.com/myendpoint',
- 'format' = 'json',
- 'gid.connector.http.sink.request.batch.size' = '50'
-)
-```
-
-#### Single submission mode
-In this mode every processed event is submitted as individual HTTP POST/PUT request.
-
-Streaming API:
-```java
-HttpSink.<String>builder()
- .setEndpointUrl("http://example.com/myendpoint")
- .setElementConverter(
- (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
- .setProperty("gid.connector.http.sink.writer.request.mode", "single")
- .build();
-```
-SQL:
-```roomsql
-CREATE TABLE http (
- id bigint,
- some_field string
-) WITH (
- 'connector' = 'http-sink',
- 'url' = 'http://example.com/myendpoint',
- 'format' = 'json',
- 'gid.connector.http.sink.writer.request.mode' = 'single'
-)
-```
-
-#### Http headers
-It is possible to set HTTP headers that will be added to HTTP request send by sink connector.
-Headers are defined via property key `gid.connector.http.sink.header.HEADER_NAME = header value` for example:
-`gid.connector.http.sink.header.X-Content-Type-Options = nosniff`.
-Properties can be set via Sink builder or Property object:
-```java
-HttpSink.<String>builder()
- .setEndpointUrl("http://example.com/myendpoint")
- .setElementConverter(
- (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
- .setProperty("gid.connector.http.sink.header.X-Content-Type-Options", "nosniff")
- .build();
-```
-or
-
-```java
-Properties properties = Properties();
-properties.setProperty("gid.connector.http.sink.header.X-Content-Type-Options", "nosniff");
-
-HttpSink.<String>builder()
- .setEndpointUrl("http://example.com/myendpoint")
- .setElementConverter(
- (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
- .setProperties(properties)
- .build();
-```
-
-In Table/SQL API, headers can be set using http sink table DDL. In example below, HTTP request done for `http` table will contain three headers:
-- `Origin`
-- `X-Content-Type-Options`
-- `Content-Type`
-
-```roomsql
-CREATE TABLE http (
- id bigint,
- some_field string
-) WITH (
- 'connector' = 'http-sink',
- 'url' = 'http://example.com/myendpoint',
- 'format' = 'json',
- 'gid.connector.http.sink.header.Origin' = '*',
- 'gid.connector.http.sink.header.X-Content-Type-Options' = 'nosniff',
- 'gid.connector.http.sink.header.Content-Type' = 'application/json'
-)
-```
-
-Note that when using OIDC, it adds an `Authentication` header with the bearer token; this will override
-an existing `Authorization` header specified in configuration.
-
-#### Custom request/response callback
-
-- Http Sink processes responses that it gets from the HTTP endpoint along their respective requests. One can customize the
-behaviour of the additional stage of processing done by Table API Sink by implementing
-[HttpPostRequestCallback](src/main/java/com/getindata/connectors/http/HttpPostRequestCallback.java) and
-[HttpPostRequestCallbackFactory](src/main/java/com/getindata/connectors/http/HttpPostRequestCallbackFactory.java)
-interfaces. Custom implementations of `HttpPostRequestCallbackFactory<HttpRequest>` can be registered along other factories in
-`resources/META-INF/services/org.apache.flink.table.factories.Factory` file and then referenced by their identifiers in
-the HttpSink DDL property field `gid.connector.http.sink.request-callback`.
-
- For example, one can create a class `CustomHttpSinkPostRequestCallbackFactory` with a unique identifier, say `rest-sink-logger`,
-that implements interface `HttpPostRequestCallbackFactory<HttpRequest>` to create a new instance of a custom callback
-`CustomHttpSinkPostRequestCallback`. This factory can be registered along other factories by appending the fully-qualified name
-of class `CustomHttpSinkPostRequestCallbackFactory` in `resources/META-INF/services/org.apache.flink.table.factories.Factory` file
-and then reference identifier `rest-sink-logger` in the HttpSink DDL property field `gid.connector.http.sink.request-callback`.
-
- A default implementation that logs those pairs as *INFO* level logs using Slf4j
-([Slf4jHttpPostRequestCallback](src/main/java/com/getindata/connectors/http/internal/table/sink/Slf4jHttpPostRequestCallback.java))
-is provided.
-
-
-- Http Lookup Source processes responses that it gets from the HTTP endpoint along their respective requests. One can customize the
-behaviour of the additional stage of processing done by Table Function API by implementing
-[HttpPostRequestCallback](src/main/java/com/getindata/connectors/http/HttpPostRequestCallback.java) and
-[HttpPostRequestCallbackFactory](src/main/java/com/getindata/connectors/http/HttpPostRequestCallbackFactory.java)
-interfaces.
-
- For example, one can create a class `CustomHttpLookupPostRequestCallbackFactory` with a unique identifier, say `rest-lookup-logger`,
-that implements interface `HttpPostRequestCallbackFactory<HttpLookupSourceRequestEntry>` to create a new instance of a custom callback
-`CustomHttpLookupPostRequestCallback`. This factory can be registered along other factories by appending the fully-qualified name
-of class `CustomHttpLookupPostRequestCallbackFactory` in `resources/META-INF/services/org.apache.flink.table.factories.Factory` file
-and then reference identifier `rest-lookup-logger` in the HTTP lookup DDL property field `gid.connector.http.source.lookup.request-callback`.
-
- A default implementation that logs those pairs as *INFO* level logs using Slf4j
-([Slf4JHttpLookupPostRequestCallback](src/main/java/com/getindata/connectors/http/internal/table/lookup/Slf4JHttpLookupPostRequestCallback.java))
-is provided.
-
-
-## HTTP status code handler
-### Sink table
-You can configure a list of HTTP status codes that should be treated as errors for HTTP sink table.
-By default all 400 and 500 response codes will be interpreted as error code.
-
-This behavior can be changed by using below properties in table definition (DDL) or passing it via `setProperty' method from Sink's builder. The property name are:
-- `gid.connector.http.sink.error.code` used to defined HTTP status code value that should be treated as error for example 404.
-Many status codes can be defined in one value, where each code should be separated with comma, for example:
-`401, 402, 403`. User can use this property also to define a type code mask. In that case, all codes from given HTTP response type will be treated as errors.
-An example of such a mask would be `3XX, 4XX, 5XX`. In this case, all 300s, 400s and 500s status codes will be treated as errors.
-- `gid.connector.http.sink.error.code.exclude` used to exclude a HTTP code from error list.
- Many status codes can be defined in one value, where each code should be separated with comma, for example:
- `401, 402, 403`. In this example, codes 401, 402 and 403 would not be interpreted as error codes.
-
-### Source table
-The source table categorizes HTTP responses into three groups based on status codes:
-- Retry codes (`gid.connector.http.source.lookup.retry-codes`):
-Responses in this group indicate a temporary issue (it can be e.g., HTTP 503 Service Unavailable). When such a response is received, the request should be retried.
-- Success codes (`gid.connector.http.source.lookup.success-codes`):
-These are expected responses that should be processed by table function.
-- Ignored responses (`gid.connector.http.source.lookup.ignored-response-codes`):
-Successful response, but its content will be ignored. For example, an HTTP 404 Not Found response is valid and indicates that the requested item does not exist, so its content can be ignored.
-- Error codes:
-Any response code that is not classified as a retry or success code falls into this category. Receiving such a response will result in a job failure.
-
-
-Above parameters support whitelisting and blacklisting. A sample configuration may look like this:
-`2XX,404,!203` - meaning all codes from group 2XX (200-299), with 404 and without 203 ('!' character). Group blacklisting e.g. !2XX is not supported.
-
-The same format is used in parameter `gid.connector.http.source.lookup.retry-codes`.
+* Unix-like environment (we use Linux, Mac OS X)
+* Git
+* Maven (we recommend version 3.8.6)
+* Java 11
-Example with explanation:
-```roomsql
-CREATE TABLE [...]
-WITH (
- [...],
- 'gid.connector.http.source.lookup.success-codes' = '2XX',
- 'gid.connector.http.source.lookup.retry-codes' = '5XX,!501,!505,!506',
- 'gid.connector.http.source.lookup.ignored-response-codes' = '404'
-)
```
-All 200s codes and 404 are considered as successful (`success-codes`, `ignored-response-codes`). These responses won't cause retry or job failure. 404 response is listed in `ignored-response-codes` parameter, what means content body will be ignored. Http with 404 code will produce just empty record.
-When server returns response with 500s code except 501, 505 and 506 then connector will re-send request based on configuration in `gid.connector.http.source.lookup.retry-strategy` parameters. By default it's fixed-delay with 1 second delay, up to 3 times per request (parameter `lookup.max-retries`). After exceeding max-retries limit the job will fail.
-A response with any other code than specified in params `success-codes` and `retry-codes` e.g. 400, 505, 301 will cause job failure.
-
-
-```roomsql
-CREATE TABLE [...]
-WITH (
- [...],
- 'gid.connector.http.source.lookup.success-codes' = '2XX',
- 'gid.connector.http.source.lookup.retry-codes' = '',
- 'gid.connector.http.source.lookup.ignored-response-codes' = '1XX,3XX,4XX,5XX'
-)
+git clone https://github.com/apache/flink-connector-http.git
+cd flink-connector-http
+mvn clean package -DskipTests
```
-In this configuration, all HTTP responses are considered successful because the sets `success-codes` and `ignored-response-codes` together cover all possible status codes. As a result, no retries will be triggered based on HTTP response codes. However, only responses with status code 200 will be parsed and processed by the Flink operator. Responses with status codes in the 1xx, 3xx, 4xx, and 5xx ranges are classified under `ignored-response-codes`.
-Note that retries remain enabled and will still occur on IOException.
-To disable retries, set `'lookup.max-retries' = '0'`.
-
-
-
-## TLS (more secure replacement for SSL) and mTLS support
-
-Both Http Sink and Lookup Source connectors support HTTPS communication using TLS 1.2 and mTLS.
-To enable Https communication simply use `https` protocol in endpoint's URL.
-
-To specify certificate(s) to be used by the server, use `gid.connector.http.security.cert.server` connector property;
-the value is a comma separated list of paths to certificate(s), for example you can use your organization's CA
-Root certificate, or a self-signed certificate.
-
-Note that if there are no security properties for a `https` url then, the JVMs default certificates are
-used - allowing use of globally recognized CAs without the need for configuration.
-
-You can also configure the connector to use mTLS. For this simply use `gid.connector.http.security.cert.client`
-and `gid.connector.http.security.key.client` connector properties to specify paths to the certificate and
-private key. The key MUST be in `PCKS8` format. Both PEM and DER keys are
-allowed.
-
-All properties can be set via Sink's builder `.setProperty(...)` method or through Sink and Source table DDL.
-
-For non production environments it is sometimes necessary to use Https connection and accept all certificates.
-In this special case, you can configure connector to trust all certificates without adding them to keystore.
-To enable this option use `gid.connector.http.security.cert.server.allowSelfSigned` property setting its value to `true`.
-
-## Basic Authentication
-The connector supports Basic Authentication using a HTTP `Authorization` header.
-The header value can be set via properties, similarly as for other headers. The connector converts the passed value to Base64 and uses it for the request.
-If the used value starts with the prefix `Basic`, or `gid.connector.http.source.lookup.use-raw-authorization-header`
-is set to `'true'`, it will be used as header value as is, without any extra modification.
-
-## OIDC Bearer Authentication
-The connector supports Bearer Authentication using a HTTP `Authorization` header. The [OAuth 2.0 rcf](https://datatracker.ietf.org/doc/html/rfc6749) mentions [Obtaining Authorization](https://datatracker.ietf.org/doc/html/rfc6749#section-4)
-and an authorization grant. OIDC makes use of this [authorisation grant](https://datatracker.ietf.org/doc/html/rfc6749#section-1.3) in a [Token Request](https://openid.net/specs/openid-connect-core-1_0.html#TokenRequest) by including a [OAuth grant type](https://oauth.net/2/grant-types/) and associated properties, the response is the [token response](https://openid.net/specs/openid-connect-core-1_0.html#TokenResponse).
-
-If you want to use this authorization then you should supply the `Token Request` body in `application/x-www-form-urlencoded` encoding
-in configuration property `gid.connector.http.security.oidc.token.request`. See [grant extension](https://datatracker.ietf.org/doc/html/rfc6749#section-4.5) for
-an example of a customised grant type token request. The supplied `token request` will be issued to the
-[token end point](https://datatracker.ietf.org/doc/html/rfc6749#section-3.2), whose url should be supplied in configuration property
-`gid.connector.http.security.oidc.token.endpoint.url`. The returned `access token` is then cached and used for subsequent requests; if the token has expired then
- a new one is requested. There is a property `gid.connector.http.security.oidc.token.expiry.reduction`, that defaults to 1 second; new tokens will
-be requested if the current time is later than the cached token expiry time minus `gid.connector.http.security.oidc.token.expiry.reduction`.
-### Restrictions at this time
-* No authentication is applied to the token request.
-* The processing does not use the refresh token if it present.
+The resulting jars can be found in the `target` directory of the respective module.
-## Table API Connector Options
-### HTTP TableLookup Source
+## Developing Flink
-| Option | Required | Description/Value [...]
-|--------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...]
-| connector | required | The Value should be set to _rest-lookup_ [...]
-| format | required | Flink's format name that should be used to decode REST response, Use `json` for a typical REST endpoint. [...]
-| url | required | The base URL that should be use for GET requests. For example _http://localhost:8080/client_ [...]
-| asyncPolling | optional | true/false - determines whether Async Polling should be used. Mechanism is based on Flink's Async I/O. [...]
-| lookup-method | optional | GET/POST/PUT (and any other) - determines what REST method should be used for lookup REST query. If not specified, `GET` method will be used. [...]
-| lookup.cache | optional | Enum possible values: `NONE`, `PARTIAL`. The cache strategy for the lookup table. Currently supports `NONE` (no caching) and `PARTIAL` (caching entries on lookup operation in external API). [...]
-| lookup.partial-cache.max-rows | optional | The max number of rows of lookup cache, over this value, the oldest rows will be expired. `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
-| lookup.partial-cache.expire-after-write | optional | The max time to live for each rows in lookup cache after writing into the cache. Specify as a [Duration](https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/deployment/config/#duration). `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
-| lookup.partial-cache.expire-after-access | optional | The max time to live for each rows in lookup cache after accessing the entry in the cache. Specify as a [Duration](https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/deployment/config/#duration). `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
-| lookup.partial-cache.cache-missing-key | optional | This is a boolean that defaults to true. Whether to store an empty value into the cache if the lookup key doesn't match any rows in the table. `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
-| lookup.max-retries | optional | The max retry times if the lookup failed; default is 3. See the following <a href="#lookup-cache">Lookup Cache</a> section for more detail. Set value 0 to disable retries. [...]
-| gid.connector.http.lookup.error.code | optional | List of HTTP status codes that should be treated as errors by HTTP Source, separated with comma. [...]
-| gid.connector.http.lookup.error.code.exclude | optional | List of HTTP status codes that should be excluded from the `gid.connector.http.lookup.error.code` list, separated with comma. [...]
-| gid.connector.http.security.cert.server | optional | Comma separated paths to trusted HTTP server certificates that should be added to the connectors trust store. [...]
-| gid.connector.http.security.cert.client | optional | Path to trusted certificate that should be used by connector's HTTP client for mTLS communication. [...]
-| gid.connector.http.security.key.client | optional | Path to trusted private key that should be used by connector's HTTP client for mTLS communication. [...]
-| gid.connector.http.security.cert.server.allowSelfSigned | optional | Accept untrusted certificates for TLS communication. [...]
-| gid.connector.http.security.oidc.token.request | optional | OIDC `Token Request` body in `application/x-www-form-urlencoded` encoding [...]
-| gid.connector.http.security.oidc.token.endpoint.url | optional | OIDC `Token Endpoint` url, to which the token request will be issued [...]
-| gid.connector.http.security.oidc.token.expiry.reduction | optional | OIDC tokens will be requested if the current time is later than the cached token expiry time minus this value. [...]
-| gid.connector.http.source.lookup.request.timeout | optional | Sets HTTP request timeout in seconds. If not specified, the default value of 30 seconds will be used. [...]
-| gid.connector.http.source.lookup.request.thread-pool.size | optional | Sets the size of pool thread for HTTP lookup request processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 8 threads will be used. [...]
-| gid.connector.http.source.lookup.response.thread-pool.size | optional | Sets the size of pool thread for HTTP lookup response processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 4 threads will be used. [...]
-| gid.connector.http.source.lookup.use-raw-authorization-header | optional | If set to `'true'`, uses the raw value set for the `Authorization` header, without transformation for Basic Authentication (base64, addition of "Basic " prefix). If not specified, defaults to `'false'`. [...]
-| gid.connector.http.source.lookup.request-callback | optional | Specify which `HttpLookupPostRequestCallback` implementation to use. By default, it is set to `slf4j-lookup-logger` corresponding to `Slf4jHttpLookupPostRequestCallback`. [...]
-| gid.connector.http.source.lookup.connection.timeout | optional | Source table connection timeout. Default - no value. [...]
-| gid.connector.http.source.lookup.success-codes | optional | Comma separated http codes considered as success response. Use [1-5]XX for groups and '!' character for excluding. [...]
-| gid.connector.http.source.lookup.retry-codes | optional | Comma separated http codes considered as transient errors. Use [1-5]XX for groups and '!' character for excluding. [...]
-| gid.connector.http.source.lookup.ignored-response-codes | optional | Comma separated http codes. Content for these responses will be ignored. Use [1-5]XX for groups and '!' character for excluding. Ignored responses togater with `gid.connector.http.source.lookup.success-codes` are considered as successful. [...]
-| gid.connector.http.source.lookup.retry-strategy.type | optional | Auto retry strategy type: fixed-delay (default) or exponential-delay. [...]
-| gid.connector.http.source.lookup.retry-strategy.fixed-delay.delay | optional | Fixed-delay interval between retries. Default 1 second. Use with`lookup.max-retries` parameter. [...]
-| gid.connector.http.source.lookup.retry-strategy.exponential-delay.initial-backoff | optional | Exponential-delay initial delay. Default 1 second. [...]
-| gid.connector.http.source.lookup.retry-strategy.exponential-delay.max-backoff | optional | Exponential-delay maximum delay. Default 1 minute. Use with `lookup.max-retries` parameter. [...]
-| gid.connector.http.source.lookup.retry-strategy.exponential-delay.backoff-multiplier | optional | Exponential-delay multiplier. Default value 1.5 [...]
-| gid.connector.http.source.lookup.proxy.host | optional | Specify the hostname of the proxy. [...]
-| gid.connector.http.source.lookup.proxy.port | optional | Specify the port of the proxy. [...]
-| gid.connector.http.source.lookup.proxy.username | optional | Specify the username used for proxy authentication. [...]
-| gid.connector.http.source.lookup.proxy.password | optional | Specify the password used for proxy authentication. [...]
-| gid.connector.http.request.query-param-fields | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The names of the fields that will be mapped to query parameters. The parameters are separated by semicolons, such as `param1;param2`. [...]
-| gid.connector.http.request.body-fields | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The names of the fields that will be mapped to the body. The parameters are separated by semicolons, such as `param1;param2`. [...]
-| gid.connector.http.request.url-map | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The map of insert names to column names used as url segments. Parses a string as a map of strings. For example if there are table columns called `customerId` and `orderId`, then specifying value `customerId:cid1,orderID:oid` and a url of https://myendpoint/customers/{cid}/orders/{oid} will mean that the url used for the lookup que [...]
+The Flink committers use IntelliJ IDEA to develop the Flink codebase.
+We recommend IntelliJ IDEA for developing projects that involve Scala code.
-### HTTP Sink
+Minimal requirements for an IDE are:
+* Support for Java and Scala (also mixed projects)
+* Support for Maven with Java and Scala
-| Option | Required | Description/Value |
-|---------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-sink'_. |
-| format | required | Specify what format to use. |
-| url | required | The base URL that should be use for HTTP requests. For example _http://localhost:8080/client_. |
-| insert-method | optional | Specify which HTTP method to use in the request. The value should be set either to `POST` or `PUT`. |
-| sink.batch.max-size | optional | Maximum number of elements that may be passed in a batch to be written downstream. |
-| sink.requests.max-inflight | optional | The maximum number of in flight requests that may exist, if any more in flight requests need to be initiated once the maximum has been reached, then it will be blocked until some have completed. |
-| sink.requests.max-buffered | optional | Maximum number of buffered records before applying backpressure. |
-| sink.flush-buffer.size | optional | The maximum size of a batch of entries that may be sent to the HTTP endpoint measured in bytes. |
-| sink.flush-buffer.timeout | optional | Threshold time in milliseconds for an element to be in a buffer before being flushed. |
-| gid.connector.http.sink.request-callback | optional | Specify which `HttpPostRequestCallback` implementation to use. By default, it is set to `slf4j-logger` corresponding to `Slf4jHttpPostRequestCallback`. |
-| gid.connector.http.sink.error.code | optional | List of HTTP status codes that should be treated as errors by HTTP Sink, separated with comma. |
-| gid.connector.http.sink.error.code.exclude | optional | List of HTTP status codes that should be excluded from the `gid.connector.http.sink.error.code` list, separated with comma. |
-| gid.connector.http.security.cert.server | optional | Path to trusted HTTP server certificate that should be add to connectors key store. More than one path can be specified using `,` as path delimiter. |
-| gid.connector.http.security.cert.client | optional | Path to trusted certificate that should be used by connector's HTTP client for mTLS communication. |
-| gid.connector.http.security.key.client | optional | Path to trusted private key that should be used by connector's HTTP client for mTLS communication. |
-| gid.connector.http.security.cert.server.allowSelfSigned | optional | Accept untrusted certificates for TLS communication. |
-| gid.connector.http.sink.request.timeout | optional | Sets HTTP request timeout in seconds. If not specified, the default value of 30 seconds will be used. |
-| gid.connector.http.sink.writer.thread-pool.size | optional | Sets the size of pool thread for HTTP Sink request processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 1 thread will be used. |
-| gid.connector.http.sink.writer.request.mode | optional | Sets Http Sink request submission mode. Two modes are available to select, `single` and `batch` which is the default mode if option is not specified. |
-| gid.connector.http.sink.request.batch.size | optional | Applicable only for `gid.connector.http.sink.writer.request.mode = batch`. Sets number of individual events/requests that will be submitted as one HTTP request by HTTP sink. The default value is 500 which is same as HTTP Sink `maxBatchSize` |
+### IntelliJ IDEA
+The IntelliJ IDE supports Maven out of the box and offers a plugin for Scala development.
-## Lookup Cache
-The HTTP Client connector can be used in lookup join as a lookup source (also known as a dimension table).
+* IntelliJ download: [https://www.jetbrains.com/idea/](https://www.jetbrains.com/idea/)
+* IntelliJ Scala Plugin: [https://plugins.jetbrains.com/plugin/?id=1347](https://plugins.jetbrains.com/plugin/?id=1347)
-By default, the lookup cache is not enabled. You can enable it by setting `lookup.cache` to `PARTIAL`.
-The scope of the cache is per job, so long-running jobs can benefit from this caching.
+Check out our [Setting up IntelliJ](https://nightlies.apache.org/flink/flink-docs-master/flinkDev/ide_setup.html#intellij-idea) guide for details.
-The lookup cache is used to improve the performance of temporal joins. By default, the lookup cache is not enabled,
-so all the API requests are sent on the network. When the lookup cache is enabled, Flink looks in the cache first,
-and only sends requests on the network when there is no cached value, then the cache is updated with the returned rows.
-The oldest rows in this cache are expired when the cache hits the max cached rows `lookup.partial-cache.max-rows`
-or when the row exceeds the max time to live specified by `lookup.partial-cache.expire-after-write`
-or `lookup.partial-cache.expire-after-access`.
+## Support
-By default, flink caches the empty query result for the primary key. You can toggle this behaviour by setting
-`lookup.partial-cache.cache-missing-key` to false.
+Don’t hesitate to ask!
+Contact the developers and community on the [mailing lists](https://flink.apache.org/community.html#mailing-lists) if you need any help.
-## Build and deployment
-To build the project locally you need to have `maven 3` and Java 11+. </br>
+[Open an issue](https://issues.apache.org/jira/browse/FLINK) if you found a bug in Flink.
-Project build command: `mvn package`. </br>
-Detailed test report can be found under `target/site/jacoco/index.xml`.
-
-## Demo application
-**Note**: This demo works only for Flink-1.15x.
-
-You can test this connector using simple mock http server provided with this repository and Flink SQL-client.
-The mock server can be started from IDE (currently only this way) by running `HttpStubApp::main` method.
-It will start HTTP server listening on `http://localhost:8080/client`
-
-Steps to follow:
-- Run Mock HTTP server from `HttpStubApp::main` method.
-- Start your Flink cluster, for example as described under https://nightlies.apache.org/flink/flink-docs-release-1.16/docs/try-flink/local_installation/
-- Start Flink SQL Client [6] by calling: `./bin/sql-client.sh -j flink-http-connector-1.0-SNAPSHOT.jar`
-- Execute SQL statements:
-Create Data Stream source Table:
-```roomsql
-CREATE TABLE Orders (id STRING, id2 STRING, proc_time AS PROCTIME()
-) WITH (
-'connector' = 'datagen',
-'rows-per-second' = '1',
-'fields.id.kind' = 'sequence',
-'fields.id.start' = '1',
-'fields.id.end' = '120',
-'fields.id2.kind' = 'sequence',
-'fields.id2.start' = '2',
-'fields.id2.end' = '120'
-);
-```
-
-Create Http Connector Lookup Table:
-```roomsql
-CREATE TABLE Customers (
- id STRING,
- id2 STRING,
- msg STRING,
- uuid STRING,
- details ROW<
- isActive BOOLEAN,
- nestedDetails ROW<
- balance STRING
- >
- >
-) WITH (
-'connector' = 'rest-lookup',
-'format' = 'json',
-'url' = 'http://localhost:8080/client',
-'asyncPolling' = 'true'
-);
-```
-
-Submit SQL Select query to join both tables:
-```roomsql
-SELECT o.id, o.id2, c.msg, c.uuid, c.isActive, c.balance FROM Orders AS o JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c ON o.id = c.id AND o.id2 = c.id2;
-```
-
-As a result, you should see a table with joined records like so:
-
-
-The `msg` column shows parameters used with REST call for given JOIN record.
-
-## Implementation
-### HTTP Source
-Implementation of an HTTP source connector is based on Flink's `TableFunction` and `AsyncTableFunction` classes.
-To be more specific we are using a `LookupTableSource`. Unfortunately Flink's new unified source interface [2] cannot be used for this type of source.
-Issue was discussed on Flink's user mailing list - https://lists.apache.org/thread/tx2w1m15zt5qnvt924mmbvr7s8rlyjmw
-
-Implementation of an HTTP Sink is based on Flink's `AsyncSinkBase` introduced in Flink 1.15 [3, 4].
-
-#### Http Response to Table schema mapping
-The mapping from Http Json Response to SQL table schema is done via Flink's Json Format [5].
+## Documentation
-## Breaking changes
-- Version 0.10
- - Http Sink submission mode changed from single to batch. From now, body of HTTP POUT/POST request will contain a Json array.
- - Changed API for public HttpSink builder. The `setHttpPostRequestCallback` expects a `PostRequestCallback`
- of generic type [HttpRequest](src/main/java/com/getindata/connectors/http/internal/sink/httpclient/HttpRequest.java)
- instead `HttpSinkRequestEntry`.
-- Version 0.20
- - Http source table parameters: `gid.connector.http.source.lookup.error.code` and `gid.connector.http.source.lookup.error.code.exclude` were removed. These parameters described http status codes which was silently ignored by source lookup table (logged only). it's not recommended to ignore all error response but it's still possible. To do this set all codes as success: `'gid.connector.http.source.lookup.success-codes' = '2XX'` with ignore body from the others responses than 200s: `'gid [...]
- - Added dependency io.github.resilience4j:resilience4j-retry
+The documentation of Apache Flink is located on the website: [https://flink.apache.org](https://flink.apache.org)
+or in the `docs/` directory of the source code.
-## TODO
+## Fork and Contribute
-### HTTP TableLookup Source
-- Check other `//TODO`'s.
+This is an active open-source project. We are always open to people who want to use the system or contribute to it.
+Contact us if you are looking for implementation tasks that fit your skills.
+This article describes [how to contribute to Apache Flink](https://flink.apache.org/contributing/how-to-contribute.html).
-### HTTP Sink
-- Make `HttpSink` retry the failed requests. Currently, it does not retry those at all, only adds their count to the `numRecordsSendErrors` metric. It should be thoroughly thought over how to do it efficiently and then implemented.
+## About
-###
-[1] https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/dev/table/sql/queries/joins/#lookup-join
-</br>
-[2] https://nightlies.apache.org/flink/flink-docs-release-1.15/docs/dev/datastream/sources/
-</br>
-[3] https://cwiki.apache.org/confluence/display/FLINK/FLIP-171%3A+Async+Sink
-</br>
-[4] https://nightlies.apache.org/flink/flink-docs-release-1.15/api/java/org/apache/flink/connector/base/sink/AsyncSinkBase.html
-</br>
-[5] https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/table/formats/json/
-</br>
-[6] https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sqlclient/
-</br>
+Apache Flink is an open source project of The Apache Software Foundation (ASF).
+The Apache Flink project originated from the [Stratosphere](http://stratosphere.eu) research project.
diff --git a/dev/README.md b/dev/README.md
deleted file mode 100644
index 96f7d69..0000000
--- a/dev/README.md
+++ /dev/null
@@ -1,36 +0,0 @@
-# Dev README
-Below are some helpful IntelliJ configurations you can set to match our coding style and standards.
-
-## Checkstyle
-This project uses checkstyle to format Java code. If developing locally, please setup checkstyle using the following steps.
-
-1. Add the CheckStyle-IDEA plugin to IntelliJ.
-- `Settings > Plugins > Marketplace > CheckStyle-IDEA > INSTALL`.
-- Restart your IDE if prompted.
-
-2. Configure IntelliJ to use the `checkstyle.xml` file provided in this directory.
-- Go to `Settings > Tools > Checkstyle` (this tool location may differ based on your version of IntelliJ).
-- Set the version to 8.29.
-- Under the `Configuration File` heading, click the `+` symbol to add our specific configuration file.
-- Give our file a useful description, such as `GID Java Checks`, and provide the `connectors/dev/checkstyle.xml` path.
-- Click `Next` to add the checkstyle file
-- Check `Active` next to it once it has been added
-- In the top right, set the Scan Scope to `Only Java sources (including tests)`
-
-3. Now, on the bottom tab bar, there should be a `CheckStyle` tab that lets you run Java style checks against using the `Check Project` button.
-
-## Java Import Order
-We use the following import order in our Java files. Please update this in `Settings > Editor > Code Style > Java > Imports > Import Layout`:
-
-```
-import java.*
-import javax.*
-<blank line>
-import scala.*
-<blank line>
-import all other imports
-<blank line>
-import com.getindata.connectors.*
-import com.getindata.connectors.internal.*
-```
-
\ No newline at end of file
diff --git a/dev/checkstyle-suppressions.xml b/dev/checkstyle-suppressions.xml
deleted file mode 100644
index 642a550..0000000
--- a/dev/checkstyle-suppressions.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!DOCTYPE suppressions PUBLIC
-"-//Puppy Crawl//DTD Suppressions 1.1//EN"
-"https://checkstyle.org/dtds/suppressions_1_1.dtd">
-
-<!--
-
- This file contains suppression rules for Checkstyle checks.
- Ideally only files that cannot be modified (e.g. third-party code)
- should be added here. All other violations should be fixed.
-
--->
-
-<suppressions>
-</suppressions>
diff --git a/dev/checkstyle.xml b/dev/checkstyle.xml
deleted file mode 100644
index ffd226b..0000000
--- a/dev/checkstyle.xml
+++ /dev/null
@@ -1,218 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!DOCTYPE module PUBLIC
- "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
- "https://checkstyle.org/dtds/configuration_1_3.dtd">
-
-<!--
-
- Checkstyle configuration based on the Google coding conventions from:
-
- - Google Java Style
- https://google.github.io/styleguide/javaguide.html
-
- with Spark-specific changes from:
-
- http://spark.apache.org/contributing.html#code-style-guide
-
- Checkstyle is very configurable. Be sure to read the documentation at
- http://checkstyle.sf.net (or in your downloaded distribution).
-
- Most Checks are configurable, be sure to consult the documentation.
-
- To completely disable a check, just comment it out or delete it from the file.
-
- Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
-
- -->
-
-<module name = "Checker">
- <property name="charset" value="UTF-8"/>
-
- <property name="severity" value="error"/>
-
- <property name="fileExtensions" value="java, properties, xml"/>
-
- <module name="SuppressionFilter">
- <property name="file" value="dev/checkstyle-suppressions.xml"/>
- </module>
-
- <!-- Checks for whitespace -->
- <!-- See http://checkstyle.sf.net/config_whitespace.html -->
- <module name="FileTabCharacter">
- <property name="eachLine" value="true"/>
- </module>
-
- <module name="RegexpSingleline">
- <!-- \s matches whitespace character, $ matches end of line. -->
- <property name="format" value="\s+$"/>
- <property name="message" value="No trailing whitespace allowed."/>
- </module>
-
- <module name="LineLength">
- <property name="max" value="120"/>
- <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
- </module>
-
- <module name="NewlineAtEndOfFile"/>
-
- <module name="TreeWalker">
- <!--
- If you wish to turn off checking for a section of code, you can put a comment in the source
- before and after the section, with the following syntax:
-
- // checkstyle.off: XXX (such as checkstyle.off: NoFinalizer)
- ... // stuff that breaks the styles
- // checkstyle.on: XXX (such as checkstyle.on: NoFinalizer)
- -->
- <module name="SuppressionCommentFilter">
- <property name="offCommentFormat" value="checkstyle\.off\: ([\w\|]+)"/>
- <property name="onCommentFormat" value="checkstyle\.on\: ([\w\|]+)"/>
- <property name="checkFormat" value="$1"/>
- </module>
- <module name="OuterTypeFilename"/>
- <module name="IllegalTokenText">
- <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
- <property name="format" value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
- <property name="message" value="Avoid using corresponding octal or Unicode escape."/>
- </module>
- <module name="AvoidEscapedUnicodeCharacters">
- <property name="allowEscapesForControlCharacters" value="true"/>
- <property name="allowByTailComment" value="true"/>
- <property name="allowNonPrintableEscapes" value="true"/>
- </module>
- <module name="NoLineWrap"/>
- <module name="EmptyBlock">
- <property name="option" value="TEXT"/>
- <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
- </module>
- <module name="NeedBraces">
- <property name="allowSingleLineStatement" value="true"/>
- </module>
- <module name="OneStatementPerLine"/>
- <module name="ArrayTypeStyle"/>
- <module name="FallThrough"/>
- <module name="UpperEll"/>
- <module name="ModifierOrder"/>
- <module name="SeparatorWrap">
- <property name="tokens" value="DOT"/>
- <property name="option" value="nl"/>
- </module>
- <module name="SeparatorWrap">
- <property name="tokens" value="COMMA"/>
- <property name="option" value="EOL"/>
- </module>
- <module name="PackageName">
- <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
- <message key="name.invalidPattern"
- value="Package name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="ClassTypeParameterName">
- <property name="format" value="([A-Z][a-zA-Z0-9]*$)"/>
- <message key="name.invalidPattern"
- value="Class type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="MethodTypeParameterName">
- <property name="format" value="([A-Z][a-zA-Z0-9]*)"/>
- <message key="name.invalidPattern"
- value="Method type name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="GenericWhitespace">
- <message key="ws.followed"
- value="GenericWhitespace ''{0}'' is followed by whitespace."/>
- <message key="ws.preceded"
- value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
- <message key="ws.illegalFollow"
- value="GenericWhitespace ''{0}'' should followed by whitespace."/>
- <message key="ws.notPreceded"
- value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
- </module>
- <module name="Indentation">
- <property name="basicOffset" value="4"/>
- <property name="braceAdjustment" value="0"/>
- <property name="caseIndent" value="4"/>
- <property name="throwsIndent" value="4"/>
- <property name="lineWrappingIndentation" value="4"/>
- <property name="arrayInitIndent" value="4"/>
- </module>
-
- <!--
- We use the following import order:
- import java.*
- import javax.*
- <blank line>
- import scala.*
- <blank line>
- import all other imports
- <blank line>
- import com.getindata.connectors.*
- import com.getindata.connectors.internal.*
- -->
- <module name="ImportOrder">
- <property name="separated" value="true"/>
- <property name="ordered" value="true"/>
- <property name="groups" value="java.,javax.,scala,*,com.getindata.connectors,com.getindata.connectors.internal"/>
- </module>
-
- <!--
- As per https://checkstyle.sourceforge.io/config_imports.html, "There is no flexibility to
- enforce empty lines between some groups and no empty lines between other groups." However,
- sometimes we want import X to come before import Y, yet we do not want them to be separated
- by a blank line. This suppression is how we achieve that.
- -->
- <!-- com.getindata.connectors and com.getindata.connectors.internal -->
- <module name="SuppressionXpathSingleFilter">
- <property name="checks" value="ImportOrder"/>
- <property name="message" value="^'com.getindata.connectors.internal\..*'.*"/>
- </module>
- <!-- java and javax -->
- <module name="SuppressionXpathSingleFilter">
- <property name="checks" value="ImportOrder"/>
- <property name="message" value="^'javax\..*'.*"/>
- </module>
-
- <module name="MethodParamPad"/>
- <module name="AnnotationLocation">
- <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
- </module>
- <module name="AnnotationLocation">
- <property name="tokens" value="VARIABLE_DEF"/>
- <property name="allowSamelineMultipleAnnotations" value="true"/>
- </module>
- <module name="MethodName">
- <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
- <message key="name.invalidPattern"
- value="Method name ''{0}'' must match pattern ''{1}''."/>
- </module>
- <module name="EmptyCatchBlock">
- <property name="exceptionVariableName" value="expected"/>
- </module>
- <module name="CommentsIndentation"/>
- <module name="UnusedImports"/>
- <module name="RedundantImport"/>
- <module name="RedundantModifier"/>
- <module name="RegexpSinglelineJava">
- <property name="format" value="throw new \w+Error\("/>
- <property name="message" value="Avoid throwing error in application code."/>
- </module>
- <module name="RegexpSinglelineJava">
- <property name="format" value="Objects\.toStringHelper"/>
- <property name="message" value="Avoid using Object.toStringHelper. Use ToStringBuilder instead." />
- </module>
- </module>
-</module>
diff --git a/docs/JoinTable.PNG b/docs/JoinTable.PNG
deleted file mode 100644
index 82379d4..0000000
Binary files a/docs/JoinTable.PNG and /dev/null differ
diff --git a/docs/content/docs/datastream/http.md b/docs/content/docs/datastream/http.md
new file mode 100644
index 0000000..030cb26
--- /dev/null
+++ b/docs/content/docs/datastream/http.md
@@ -0,0 +1,179 @@
+---
+title: HTTP
+weight: 3
+type: docs
+aliases:
+ - /dev/connectors/http.html
+ - /apis/streaming/connectors/http.html
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache HTTP Connector
+The HTTP connector allows for pulling data from external system via HTTP methods and HTTP Sink that allows for sending data to external system via HTTP requests.
+
+Note this connector was donated to Flink in [FLIP-532](https://cwiki.apache.org/confluence/display/FLINK/FLIP-532%3A+Donate+GetInData+HTTP+Connector+to+Flink).
+Existing java applications built using the original repository will need to be recompiled to pick up the new flink package names.
+
+The HTTP sink connector supports the Flink streaming API.
+
+<!-- TOC -->
+* [Apache HTTP Connector](#apache-http-connector)
+ * [Working with HTTP sink Flink streaming API](#working-with-http-sink-flink-streaming-api)
+ * [Sink Connector options](#sink-connector-options-)
+ * [Batch submission mode](#batch-submission-mode)
+ * [Single submission mode](#single-submission-mode)
+ * [Http headers](#http-headers)
+ * [TLS (more secure replacement for SSL) and mTLS support](#tls-more-secure-replacement-for-ssl-and-mtls-support)
+ * [Basic Authentication](#basic-authentication)
+ * [OIDC Bearer Authentication](#oidc-bearer-authentication)
+ * [Restrictions at this time](#restrictions-at-this-time)
+<!-- TOC -->
+## Working with HTTP sink Flink streaming API
+
+In order to change submission batch size use `flink.connector.http.sink.request.batch.size` property. For example:
+
+### Sink Connector options
+These option are specified on the builder using the setProperty method.
+
+| Option | Required | Description/Value |
+|---------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-sink'_. |
+| format | required | Specify what format to use. |
+| url | required | The base URL that should be use for HTTP requests. For example _http://localhost:8080/client_. |
+| insert-method | optional | Specify which HTTP method to use in the request. The value should be set either to `POST` or `PUT`. |
+| sink.batch.max-size | optional | Maximum number of elements that may be passed in a batch to be written downstream. |
+| sink.requests.max-inflight | optional | The maximum number of in flight requests that may exist, if any more in flight requests need to be initiated once the maximum has been reached, then it will be blocked until some have completed. |
+| sink.requests.max-buffered | optional | Maximum number of buffered records before applying backpressure. |
+| sink.flush-buffer.size | optional | The maximum size of a batch of entries that may be sent to the HTTP endpoint measured in bytes. |
+| sink.flush-buffer.timeout | optional | Threshold time in milliseconds for an element to be in a buffer before being flushed. |
+| flink.connector.http.sink.request-callback | optional | Specify which `HttpPostRequestCallback` implementation to use. By default, it is set to `slf4j-logger` corresponding to `Slf4jHttpPostRequestCallback`. |
+| flink.connector.http.sink.error.code | optional | List of HTTP status codes that should be treated as errors by HTTP Sink, separated with comma. |
+| flink.connector.http.sink.error.code.exclude | optional | List of HTTP status codes that should be excluded from the `flink.connector.http.sink.error.code` list, separated with comma. |
+| flink.connector.http.security.cert.server | optional | Path to trusted HTTP server certificate that should be add to connectors key store. More than one path can be specified using `,` as path delimiter. |
+| flink.connector.http.security.cert.client | optional | Path to trusted certificate that should be used by connector's HTTP client for mTLS communication. |
+| flink.connector.http.security.key.client | optional | Path to trusted private key that should be used by connector's HTTP client for mTLS communication. |
+| flink.connector.http.security.cert.server.allowSelfSigned | optional | Accept untrusted certificates for TLS communication. |
+| flink.connector.http.sink.request.timeout | optional | Sets HTTP request timeout in seconds. If not specified, the default value of 30 seconds will be used. |
+| flink.connector.http.sink.writer.thread-pool.size | optional | Sets the size of pool thread for HTTP Sink request processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 1 thread will be used. |
+| flink.connector.http.sink.writer.request.mode | optional | Sets Http Sink request submission mode. Two modes are available to select, `single` and `batch` which is the default mode if option is not specified. |
+| flink.connector.http.sink.request.batch.size | optional | Applicable only for `flink.connector.http.sink.writer.request.mode = batch`. Sets number of individual events/requests that will be submitted as one HTTP request by HTTP sink. The default value is 500 which is same as HTTP Sink `maxBatchSize` |
+
+
+
+### Batch submission mode
+
+By default, batch size is set to 500 which is the same as Http Sink's `maxBatchSize` property and has value of 500.
+The `maxBatchSize' property sets maximal number of events that will by buffered by Flink runtime before passing it to Http Sink for processing.
+
+Streaming API:
+```java
+HttpSink.<String>builder()
+ .setEndpointUrl("http://example.com/myendpoint")
+ .setElementConverter(
+ (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
+ .setProperty("http.sink.request.batch.size", "50")
+ .build();
+```
+
+### Single submission mode
+In this mode every processed event is submitted as individual HTTP POST/PUT request.
+
+Streaming API:
+```java
+HttpSink.<String>builder()
+ .setEndpointUrl("http://example.com/myendpoint")
+ .setElementConverter(
+ (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
+ .setProperty("http.sink.writer.request.mode", "single")
+ .build();
+```
+### Http headers
+It is possible to set HTTP headers that will be added to HTTP request send by sink connector.
+Headers are defined via property key `flink.connector.http.sink.header.HEADER_NAME = header value` for example:
+`flink.connector.http.sink.header.X-Content-Type-Options = nosniff`.
+Properties can be set via Sink builder or Property object:
+```java
+HttpSink.<String>builder()
+ .setEndpointUrl("http://example.com/myendpoint")
+ .setElementConverter(
+ (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
+ .setProperty("http.sink.header.X-Content-Type-Options", "nosniff")
+ .build();
+```
+or
+
+```java
+Properties properties = Properties();
+properties.setProperty("http.sink.header.X-Content-Type-Options", "nosniff");
+
+HttpSink.<String>builder()
+ .setEndpointUrl("http://example.com/myendpoint")
+ .setElementConverter(
+ (s, _context) -> new HttpSinkRequestEntry("POST", s.getBytes(StandardCharsets.UTF_8)))
+ .setProperties(properties)
+ .build();
+```
+
+
+## TLS (more secure replacement for SSL) and mTLS support
+
+Both Http Sink and Lookup Source connectors support HTTPS communication using TLS 1.2 and mTLS.
+To enable Https communication simply use `https` protocol in endpoint's URL.
+
+To specify certificate(s) to be used by the server, use `flink.connector.http.security.cert.server` connector property;
+the value is a comma separated list of paths to certificate(s), for example you can use your organization's CA
+Root certificate, or a self-signed certificate.
+
+Note that if there are no security properties for a `https` url then, the JVMs default certificates are
+used - allowing use of globally recognized CAs without the need for configuration.
+
+You can also configure the connector to use mTLS. For this simply use `flink.connector.http.security.cert.client`
+and `flink.connector.http.security.key.client` connector properties to specify paths to the certificate and
+private key. The key MUST be in `PCKS8` format. Both PEM and DER keys are
+allowed.
+
+All properties can be set via Sink's builder `.setProperty(...)` method or through Sink and Source table DDL.
+
+For non production environments it is sometimes necessary to use Https connection and accept all certificates.
+In this special case, you can configure connector to trust all certificates without adding them to keystore.
+To enable this option use `flink.connector.http.security.cert.server.allowSelfSigned` property setting its value to `true`.
+
+## Basic Authentication
+The connector supports Basic Authentication using a HTTP `Authorization` header.
+The header value can be set via properties, similarly as for other headers. The connector converts the passed value to Base64 and uses it for the request.
+If the used value starts with the prefix `Basic`, or `flink.connector.http.source.lookup.use-raw-authorization-header`
+is set to `'true'`, it will be used as header value as is, without any extra modification.
+
+## OIDC Bearer Authentication
+The connector supports Bearer Authentication using a HTTP `Authorization` header. The [OAuth 2.0 rcf](https://datatracker.ietf.org/doc/html/rfc6749) mentions [Obtaining Authorization](https://datatracker.ietf.org/doc/html/rfc6749#section-4)
+and an authorization grant. OIDC makes use of this [authorisation grant](https://datatracker.ietf.org/doc/html/rfc6749#section-1.3) in a [Token Request](https://openid.net/specs/openid-connect-core-1_0.html#TokenRequest) by including a [OAuth grant type](https://oauth.net/2/grant-types/) and associated properties, the response is the [token response](https://openid.net/specs/openid-connect-core-1_0.html#TokenResponse).
+
+If you want to use this authorization then you should supply the `Token Request` body in `application/x-www-form-urlencoded` encoding
+in configuration property `flink.connector.http.security.oidc.token.request`. See [grant extension](https://datatracker.ietf.org/doc/html/rfc6749#section-4.5) for
+an example of a customised grant type token request. The supplied `token request` will be issued to the
+[token end point](https://datatracker.ietf.org/doc/html/rfc6749#section-3.2), whose url should be supplied in configuration property
+`flink.connector.http.security.oidc.token.endpoint.url`. The returned `access token` is then cached and used for subsequent requests; if the token has expired then
+a new one is requested. There is a property `flink.connector.http.security.oidc.token.expiry.reduction`, that defaults to 1 second; new tokens will
+be requested if the current time is later than the cached token expiry time minus `flink.connector.http.security.oidc.token.expiry.reduction`.
+
+### Restrictions at this time
+* No authentication is applied to the token request.
+* The processing does not use the refresh token if it present.
+ {{< top >}}
diff --git a/docs/content/docs/table/http.md b/docs/content/docs/table/http.md
new file mode 100644
index 0000000..630c0f2
--- /dev/null
+++ b/docs/content/docs/table/http.md
@@ -0,0 +1,545 @@
+
+---
+title: HTTP
+weight: 3
+type: docs
+aliases:
+- /dev/table/connectors/http.html
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# HTTP Connector
+
+{{< label "Sink: Streaming Append Mode" >}}
+{{< label "Lookup Source: Sync Mode" >}}
+{{< label "Lookup Source: Async Mode" >}}
+{{< label "Sink: Batch" >}}
+
+
+The HTTP connector allows for pulling data from external system via HTTP methods and HTTP Sink that allows for sending data to external system via HTTP requests.
+
+The HTTP source connector supports [Lookup Joins](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sourcessinks/#lookup-table-source) in [Table API and SQL](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/overview/).
+
+<!-- TOC -->
+* [HTTP Connector](#http-connector)
+ * [Dependencies](#dependencies)
+ * [Working with HTTP lookup source tables](#working-with-http-lookup-source-tables)
+ * [HTTP Lookup Table API and SQL Source example](#http-lookup-table-api-and-sql-source-example)
+ * [Using a HTTP Lookup Source in a lookup join](#using-a-http-lookup-source-in-a-lookup-join)
+ * [Lookup Source Connector Options](#lookup-source-connector-options)
+ * [Query Creators](#query-creators)
+ * [generic-json-url Query Creator](#generic-json-url-query-creator)
+ * [generic-json-url Query Creator](#generic-json-url-query-creator-1)
+ * [Http headers](#http-headers)
+ * [Timeouts](#timeouts)
+ * [Source table HTTP status code](#source-table-http-status-code)
+ * [Retries (Lookup source)](#retries-lookup-source)
+ * [Retry strategy](#retry-strategy)
+ * [Lookup multiple results](#lookup-multiple-results)
+ * [Working with HTTP sink tables](#working-with-http-sink-tables)
+ * [HTTP Sink](#http-sink)
+ * [Sink Connector Options](#sink-connector-options)
+ * [Sink table HTTP status codes](#sink-table-http-status-codes)
+ * [Request submission](#request-submission)
+ * [Batch submission mode](#batch-submission-mode)
+ * [Single submission mode](#single-submission-mode)
+ * [Available Metadata](#available-metadata)
+ * [HTTP status code handler](#http-status-code-handler)
+ * [Security considerations](#security-considerations)
+ * [TLS (more secure replacement for SSL) and mTLS support](#tls-more-secure-replacement-for-ssl-and-mtls-support)
+ * [Basic Authentication](#basic-authentication)
+ * [OIDC Bearer Authentication](#oidc-bearer-authentication)
+ * [Restrictions at this time](#restrictions-at-this-time)
+<!-- TOC -->
+## Dependencies
+
+{{< sql_connector_download_table "http" >}}
+
+The HTTP connector is not part of the binary distribution.
+See how to link with it for cluster execution [here]({{< ref "docs/dev/configuration/overview" >}}).
+
+## Migration from GetInData HTTP connector
+
+The GetInData HTTP connector was donated to Flink in [FLIP-532](https://cwiki.apache.org/confluence/display/FLINK/FLIP-532%3A+Donate+GetInData+HTTP+Connector+to+Flink). The Flink connector has the same capabilities as the original connector.
+The Flink connector does have some changes that you need to be aware of if you are migrating from using the original connector:
+
+* Existing java applications will need to be recompiled to pick up the new flink package names.
+* Existing application and SQL need to be amended to use the new connector option names. The new option names do not have
+the _com.getindata.http_ prefix, the prefix is now _http_ prefix.
+
+## Working with HTTP lookup source tables
+
+### HTTP Lookup Table API and SQL Source example
+Here is an example Flink SQL Enrichment Lookup Table definition:
+
+```roomsql
+CREATE TABLE Customers (
+ id STRING,
+ id2 STRING,
+ msg STRING,
+ uuid STRING,
+ details ROW<
+ isActive BOOLEAN,
+ nestedDetails ROW<
+ balance STRING
+ >
+ >
+) WITH (
+'connector' = 'rest-lookup',
+'format' = 'json',
+'url' = 'http://localhost:8080/client',
+'asyncPolling' = 'true'
+)
+```
+
+### Using a HTTP Lookup Source in a lookup join
+
+To easy see how the lookup enrichment works, we can define a data source using datagen:
+```roomsql
+CREATE TABLE Orders (
+ id STRING,
+ id2 STRING,
+ proc_time AS PROCTIME()
+) WITH (
+'connector' = 'datagen',
+'rows-per-second' = '1',
+'fields.id.kind' = 'sequence',
+'fields.id.start' = '1',
+'fields.id.end' = '120',
+'fields.id2.kind' = 'sequence',
+'fields.id2.start' = '2',
+'fields.id2.end' = '120'
+);
+```
+
+Then we can enrich the _Orders_ table with the _Customers_ HTTP table with the following SQL:
+
+```roomsql
+SELECT o.id, o.id2, c.msg, c.uuid, c.isActive, c.balance FROM Orders AS o
+JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c ON o.id = c.id AND o.id2 = c.id2
+```
+
+The columns and their values used for JOIN `ON` condition will be used as HTTP GET parameters where the column name will be used as a request parameter name.
+
+For Example:
+``
+http://localhost:8080/client/service?id=1&uuid=2
+``
+
+Or for REST POST method they will be converted to Json and used as request body. In this case, json request body will look like this:
+```json
+{
+ "id": "1",
+ "uuid": "2"
+}
+```
+
+### Lookup Source Connector Options
+
+Note the options with the prefix _http_ are the HTTP connector specific options, the others are Flink options.
+
+| Option | Required | Description/Value [...]
+|:-----------------------------------------------------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...]
+| connector | required | The Value should be set to _rest-lookup_ [...]
+| format | required | Flink's format name that should be used to decode REST response, Use `json` for a typical REST endpoint. [...]
+| url | required | The base URL that should be use for GET requests. For example _http://localhost:8080/client_ [...]
+| asyncPolling | optional | true/false - determines whether Async Polling should be used. Mechanism is based on Flink's Async I/O. [...]
+| lookup-method | optional | GET/POST/PUT (and any other) - determines what REST method should be used for lookup REST query. If not specified, `GET` method will be used. [...]
+| lookup.cache | optional | Enum possible values: `NONE`, `PARTIAL`. The cache strategy for the lookup table. Currently supports `NONE` (no caching) and `PARTIAL` (caching entries on lookup operation in external API). [...]
+| lookup.partial-cache.max-rows | optional | The max number of rows of lookup cache, over this value, the oldest rows will be expired. `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
+| lookup.partial-cache.expire-after-write | optional | The max time to live for each rows in lookup cache after writing into the cache. Specify as a [Duration](https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/deployment/config/#duration). `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
+| lookup.partial-cache.expire-after-access | optional | The max time to live for each rows in lookup cache after accessing the entry in the cache. Specify as a [Duration](https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/deployment/config/#duration). `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
+| lookup.partial-cache.cache-missing-key | optional | This is a boolean that defaults to true. Whether to store an empty value into the cache if the lookup key doesn't match any rows in the table. `lookup.cache` must be set to `PARTIAL` to use this option. See the following <a href="#lookup-cache">Lookup Cache</a> section for more details. [...]
+| lookup.max-retries | optional | The max retry times if the lookup failed; default is 3. See the following <a href="#lookup-cache">Lookup Cache</a> section for more detail. Set value 0 to disable retries. [...]
+| lookup.error.code | optional | List of HTTP status codes that should be treated as errors by HTTP Source, separated with comma. [...]
+| lookup.error.code.exclude | optional | List of HTTP status codes that should be excluded from the `http.lookup.error.code` list, separated with comma. [...]
+| http.security.cert.server | optional | Comma separated paths to trusted HTTP server certificates that should be added to the connectors trust store. [...]
+| http.security.cert.client | optional | Path to trusted certificate that should be used by connector's HTTP client for mTLS communication. [...]
+| http.security.key.client | optional | Path to trusted private key that should be used by connector's HTTP client for mTLS communication. [...]
+| http.security.cert.server.allowSelfSigned | optional | Accept untrusted certificates for TLS communication. [...]
+| http.security.oidc.token.request | optional | OIDC `Token Request` body in `application/x-www-form-urlencoded` encoding [...]
+| http.security.oidc.token.endpoint.url | optional | OIDC `Token Endpoint` url, to which the token request will be issued [...]
+| http.security.oidc.token.expiry.reduction | optional | OIDC tokens will be requested if the current time is later than the cached token expiry time minus this value. [...]
+| http.source.lookup.request.timeout | optional | Sets HTTP request timeout in seconds. If not specified, the default value of 30 seconds will be used. [...]
+| http.source.lookup.request.thread-pool.size | optional | Sets the size of pool thread for HTTP lookup request processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 8 threads will be used. [...]
+| http.source.lookup.response.thread-pool.size | optional | Sets the size of pool thread for HTTP lookup response processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 4 threads will be used. [...]
+| http.source.lookup.use-raw-authorization-header | optional | If set to `'true'`, uses the raw value set for the `Authorization` header, without transformation for Basic Authentication (base64, addition of "Basic " prefix). If not specified, defaults to `'false'`. [...]
+| http.source.lookup.request-callback | optional | Specify which `HttpLookupPostRequestCallback` implementation to use. By default, it is set to `slf4j-lookup-logger` corresponding to `Slf4jHttpLookupPostRequestCallback`. [...]
+| http.source.lookup.connection.timeout | optional | Source table connection timeout. Default - no value. [...]
+| http.source.lookup.success-codes | optional | Comma separated http codes considered as success response. Use [1-5]XX for groups and '!' character for excluding. [...]
+| http.source.lookup.retry-codes | optional | Comma separated http codes considered as transient errors. Use [1-5]XX for groups and '!' character for excluding. [...]
+| http.source.lookup.ignored-response-codes | optional | Comma separated http codes. Content for these responses will be ignored. Use [1-5]XX for groups and '!' character for excluding. Ignored responses togater with `http.source.lookup.success-codes` are considered as successful. [...]
+| http.source.lookup.retry-strategy.type | optional | Auto retry strategy type: fixed-delay (default) or exponential-delay. [...]
+| http.source.lookup.retry-strategy.fixed-delay.delay | optional | Fixed-delay interval between retries. Default 1 second. Use with`lookup.max-retries` parameter. [...]
+| http.source.lookup.retry-strategy.exponential-delay.initial-backoff | optional | Exponential-delay initial delay. Default 1 second. [...]
+| http.source.lookup.retry-strategy.exponential-delay.max-backoff | optional | Exponential-delay maximum delay. Default 1 minute. Use with `lookup.max-retries` parameter. [...]
+| http.source.lookup.retry-strategy.exponential-delay.backoff-multiplier | optional | Exponential-delay multiplier. Default value 1.5 [...]
+| http.source.lookup.proxy.host | optional | Specify the hostname of the proxy. [...]
+| http.source.lookup.proxy.port | optional | Specify the port of the proxy. [...]
+| http.source.lookup.proxy.username | optional | Specify the username used for proxy authentication. [...]
+| http.source.lookup.proxy.password | optional | Specify the password used for proxy authentication. [...]
+| http.request.query-param-fields | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The names of the fields that will be mapped to query parameters. The parameters are separated by semicolons, such as `param1;param2`. [...]
+| http.request.body-fields | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The names of the fields that will be mapped to the body. The parameters are separated by semicolons, such as `param1;param2`. [...]
+| http.request.url-map | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The map of insert names to column names used as url segments. Parses a string as a map of strings. For example if there are table columns called `customerId` and `orderId`, then specifying value `customerId:cid1,orderID:oid` and a url of https://myendpoint/customers/{cid}/orders/{oid} will mean that the url used for the lookup query will dynami [...]
+
+### Query Creators
+
+In the above example we see that HTTP GET operations and HTTP POST operations result in different mapping of the columns to the
+HTTP request content. In reality, you will want to have move control over how the SQL columns are mapped to the HTTP content.
+The HTTP connector supplies a number of Query Creators that you can use define these mappings.
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 76%">Name</th>
+ <th class="text-center" style="width: 8%">Query param mapping</th>
+ <th class="text-center" style="width: 8%">URL path mapping</th>
+ <th class="text-center" style="width: 8%">Body mapping</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><h5>generic-json-url</h5></td>
+ <td>✓</td>
+ <td>✓</td>
+ <td>✓</td>
+ </tr>
+ <tr>
+ <td><h5>generic-get-query</h5></td>
+ <td><✓ for GETs/td>
+ <td></td>
+ <td>✓ for PUTs and POSTs</td>
+ </tr>
+
+ </tbody>
+</table>
+
+### generic-json-url Query Creator
+
+The recommended Query creator for json is called _generic-json-url_, which allows column content to be mapped as URL, path, body and query parameter request values; it supports
+POST, PUT and GET operations. This query creator allows you to issue json requests without needing to code
+your own custom http connector. The mappings from columns to the json request are supplied in the query creator configuration
+parameters `http.request.query-param-fields`, `http.request.body-fields` and `http.request.url-map`.
+
+### generic-json-url Query Creator
+
+The default Query Creator is called _generic-json-url_. For body based queries such as POST/PUT requests, the
+([GenericGetQueryCreator](flink-connector-http/src/main/java/org/apache/flink/connectors/http/internal/table/lookup/querycreators/GenericGetQueryCreator.java))is provided as a default query creator. This implementation uses Flink's [json-format](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/table/formats/json/) to convert RowData object into Json String.
+For GET requests can be used for query parameter based queries.
+
+The _generic-json-url_ allows for using custom formats that will perform serialization to Json. Thanks to this, users can create their own logic for converting RowData to Json Strings suitable for their HTTP endpoints and use this logic as custom format
+with HTTP Lookup connector and SQL queries.
+To create a custom format user has to implement Flink's `SerializationSchema` and `SerializationFormatFactory` interfaces and register custom format factory along other factories in
+`resources/META-INF.services/org.apache.flink.table.factories.Factory` file. This is common Flink mechanism for providing custom implementations for various factories.
+
+### Http headers
+It is possible to set HTTP headers that will be added to HTTP request send by lookup source connector.
+Headers are defined via property key `http.source.lookup.header.HEADER_NAME = header value` for example:
+`http.source.lookup.header.X-Content-Type-Options = nosniff`.
+
+Headers can be set using http lookup source table DDL. In example below, HTTP request done for `http-lookup` table will contain three headers:
+- `Origin`
+- `X-Content-Type-Options`
+- `Content-Type`
+
+```roomsql
+CREATE TABLE http-lookup (
+ id bigint,
+ some_field string
+) WITH (
+ 'connector' = 'rest-lookup',
+ 'format' = 'json',
+ 'url' = 'http://localhost:8080/client',
+ 'asyncPolling' = 'true',
+ 'http.source.lookup.header.Origin' = '*',
+ 'http.source.lookup.header.X-Content-Type-Options' = 'nosniff',
+ 'http.source.lookup.header.Content-Type' = 'application/json'
+)
+```
+
+Note that when using OIDC, it adds an `Authentication` header with the bearer token; this will override
+an existing `Authorization` header specified in configuration.
+
+### Timeouts
+Lookup Source is guarded by two timeout timers. First one is specified by Flink's AsyncIO operator that executes `AsyncTableFunction`.
+The default value of this timer is set to 3 minutes and can be changed via `table.exec.async-lookup.timeout` [option](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/config/#table-exec-async-lookup-timeout).
+
+The second one is set per individual HTTP requests by HTTP client. Its default value is set currently to 30 seconds and can be changed via `http.source.lookup.request.timeout` option.
+
+Flink's current implementation of `AsyncTableFunction` does not allow specifying custom logic for handling Flink AsyncIO timeouts as it is for Java API.
+Because of that, if AsyncIO timer passes, Flink will throw TimeoutException which will cause job restart.
+
+### Source table HTTP status code
+The source table categorizes HTTP responses into three groups based on status codes:
+- Retry codes (`http.source.lookup.retry-codes`):
+ Responses in this group indicate a temporary issue (it can be e.g., HTTP 503 Service Unavailable). When such a response is received, the request should be retried.
+- Success codes (`http.source.lookup.success-codes`):
+ These are expected responses that should be processed by table function.
+- Ignored responses (`http.source.lookup.ignored-response-codes`):
+ Successful response, but its content will be ignored. For example, an HTTP 404 Not Found response is valid and indicates that the requested item does not exist, so its content can be ignored.
+- Error codes:
+ Any response code that is not classified as a retry or success code falls into this category. Receiving such a response will result in a job failure.
+
+
+### Retries (Lookup source)
+Lookup source handles auto-retries for two scenarios:
+1. IOException occurs (e.g. temporary network outage)
+2. The response contains a HTTP error code that indicates a retriable error. These codes are defined in the table configuration (see `http.source.lookup.retry-codes`).
+ Retries are executed silently, without restarting the job. After reaching max retries attempts (per request) operation will fail and restart job.
+
+Notice that HTTP codes are categorized into into 3 groups:
+- successful responses - response is returned immediately for further processing
+- temporary errors - request will be retried up to the retry limit
+- error responses - unexpected responses are not retried and will fail the job. Any HTTP error code which is not configured as successful or temporary error is treated as an unretriable error.
+
+##### Retry strategy
+User can choose retry strategy type for source table:
+- fixed-delay - http request will be re-sent after specified delay.
+- exponential-delay - request will be re-sent with exponential backoff strategy, limited by `lookup.max-retries` attempts. The delay for each retry is calculated as the previous attempt's delay multiplied by the backoff multiplier (parameter `http.source.lookup.retry-strategy.exponential-delay.backoff-multiplier`) up to `http.source.lookup.retry-strategy.exponential-delay.max-backoff`. The initial delay value is defined in the table configuration as `http.source.lookup.retry-strategy.exp [...]
+
+
+#### Lookup multiple results
+
+Typically, join can return zero, one or more results. What is more, there are lots of possible REST API designs and
+pagination methods. Currently, the connector supports only two simple approaches (`http.source.lookup.result-type`):
+
+- `single-value` - REST API returns single object.
+- `array` - REST API returns array of objects. Pagination is not supported yet.
+
+## Working with HTTP sink tables
+
+### HTTP Sink
+The following example shows the minimum Table API example to create a sink:
+
+```roomsql
+CREATE TABLE http (
+ id bigint,
+ some_field string
+) WITH (
+ 'connector' = 'http-sink',
+ 'url' = 'http://example.com/myendpoint',
+ 'format' = 'json'
+)
+```
+
+Then use `INSERT` SQL statement to send data to your HTTP endpoint:
+
+```roomsql
+INSERT INTO http VALUES (1, 'Ninette'), (2, 'Hedy')
+```
+
+When `'format' = 'json'` is specified on the table definition, the HTTP sink sends json payloads. It is possible to change the format of the payload by specifying
+another format name.
+
+### Sink Connector Options
+
+| Option | Required | Description/Value |
+|-------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| connector | required | Specify what connector to use. For HTTP Sink it should be set to _'http-sink'_. |
+| format | required | Specify what format to use. |
+| url | required | The base URL that should be use for HTTP requests. For example _http://localhost:8080/client_. |
+| insert-method | optional | Specify which HTTP method to use in the request. The value should be set either to `POST` or `PUT`. |
+| sink.batch.max-size | optional | Maximum number of elements that may be passed in a batch to be written downstream. |
+| sink.requests.max-inflight | optional | The maximum number of in flight requests that may exist, if any more in flight requests need to be initiated once the maximum has been reached, then it will be blocked until some have completed. |
+| sink.requests.max-buffered | optional | Maximum number of buffered records before applying backpressure. |
+| sink.flush-buffer.size | optional | The maximum size of a batch of entries that may be sent to the HTTP endpoint measured in bytes. |
+| sink.flush-buffer.timeout | optional | Threshold time in milliseconds for an element to be in a buffer before being flushed. |
+| http.sink.request-callback | optional | Specify which `HttpPostRequestCallback` implementation to use. By default, it is set to `slf4j-logger` corresponding to `Slf4jHttpPostRequestCallback`. |
+| http.sink.error.code | optional | List of HTTP status codes that should be treated as errors by HTTP Sink, separated with comma. |
+| http.sink.error.code.exclude | optional | List of HTTP status codes that should be excluded from the `http.sink.error.code` list, separated with comma. |
+| http.security.cert.server | optional | Path to trusted HTTP server certificate that should be add to connectors key store. More than one path can be specified using `,` as path delimiter. |
+| http.security.cert.client | optional | Path to trusted certificate that should be used by connector's HTTP client for mTLS communication. |
+| http.security.key.client | optional | Path to trusted private key that should be used by connector's HTTP client for mTLS communication. |
+| http.security.cert.server.allowSelfSigned | optional | Accept untrusted certificates for TLS communication. |
+| http.sink.request.timeout | optional | Sets HTTP request timeout in seconds. If not specified, the default value of 30 seconds will be used. |
+| http.sink.writer.thread-pool.size | optional | Sets the size of pool thread for HTTP Sink request processing. Increasing this value would mean that more concurrent requests can be processed in the same time. If not specified, the default value of 1 thread will be used. |
+| http.sink.writer.request.mode | optional | Sets Http Sink request submission mode. Two modes are available to select, `single` and `batch` which is the default mode if option is not specified. |
+| http.sink.request.batch.size | optional | Applicable only for `http.sink.writer.request.mode = batch`. Sets number of individual events/requests that will be submitted as one HTTP request by HTTP sink. The default value is 500 which is same as HTTP Sink `maxBatchSize` |
+
+### Sink table HTTP status codes
+You can configure a list of HTTP status codes that should be treated as errors for HTTP sink table.
+By default all 400 and 500 response codes will be interpreted as error code.
+
+This behavior can be changed by using below properties in table definition. The property name are:
+- `http.sink.error.code` used to defined HTTP status code value that should be treated as error for example 404.
+ Many status codes can be defined in one value, where each code should be separated with comma, for example:
+ `401, 402, 403`. User can use this property also to define a type code mask. In that case, all codes from given HTTP response type will be treated as errors.
+ An example of such a mask would be `3XX, 4XX, 5XX`. In this case, all 300s, 400s and 500s status codes will be treated as errors.
+- `http.sink.error.code.exclude` used to exclude a HTTP code from error list.
+ Many status codes can be defined in one value, where each code should be separated with comma, for example:
+ `401, 402, 403`. In this example, codes 401, 402 and 403 would not be interpreted as error codes.
+
+
+### Request submission
+HTTP Sink by default submits events in batch. The submission mode can be changed using `http.sink.writer.request.mode` property using `single` or `batch` as property value.
+
+### Batch submission mode
+In batch mode, a number of events (processed elements) will be batched and submitted in one HTTP request.
+In this mode, HTTP PUT/POST request's body contains a Json array, where every element of this array represents
+individual event.
+
+An example of Http Sink batch request body containing data for three events:
+```json
+[
+ {
+ "id": 1,
+ "first_name": "Ninette",
+ "last_name": "Clee",
+ "gender": "Female",
+ "stock": "CDZI",
+ "currency": "RUB",
+ "tx_date": "2021-08-24 15:22:59"
+ },
+ {
+ "id": 2,
+ "first_name": "Rob",
+ "last_name": "Zombie",
+ "gender": "Male",
+ "stock": "DGICA",
+ "currency": "GBP",
+ "tx_date": "2021-10-25 20:53:54"
+ },
+ {
+ "id": 3,
+ "first_name": "Adam",
+ "last_name": "Jones",
+ "gender": "Male",
+ "stock": "DGICA",
+ "currency": "PLN",
+ "tx_date": "2021-10-26 20:53:54"
+ }
+]
+```
+
+By default, batch size is set to 500 which is the same as Http Sink's `maxBatchSize` property and has value of 500.
+The `maxBatchSize' property sets maximal number of events that will by buffered by Flink runtime before passing it to Http Sink for processing.
+
+```roomsql
+CREATE TABLE http (
+ id bigint,
+ some_field string
+) WITH (
+ 'connector' = 'http-sink',
+ 'url' = 'http://example.com/myendpoint',
+ 'format' = 'json',
+ 'http.sink.request.batch.size' = '50'
+)
+```
+
+### Single submission mode
+In this mode every processed event is submitted as individual HTTP POST/PUT request.
+
+SQL:
+```roomsql
+CREATE TABLE http (
+ id bigint,
+ some_field string
+) WITH (
+ 'connector' = 'http-sink',
+ 'url' = 'http://example.com/myendpoint',
+ 'format' = 'json',
+ 'http.sink.writer.request.mode' = 'single'
+)
+```
+
+## Available Metadata
+
+The is no available metadata for this connector.
+
+## HTTP status code handler
+
+Above parameters support include lists and exclude lists. A sample configuration may look like this:
+`2XX,404,!203` - meaning all codes from group 2XX (200-299), with 404 and without 203 ('!' character). Group exclude listing e.g. !2XX is not supported.
+
+The same format is used in parameter `http.source.lookup.retry-codes`.
+
+Example with explanation:
+```roomsql
+CREATE TABLE [...]
+WITH (
+ [...],
+ 'http.source.lookup.success-codes' = '2XX',
+ 'http.source.lookup.retry-codes' = '5XX,!501,!505,!506',
+ 'http.source.lookup.ignored-response-codes' = '404'
+)
+```
+All 200s codes and 404 are considered as successful (`success-codes`, `ignored-response-codes`). These responses won't cause retry or job failure. 404 response is listed in `ignored-response-codes` parameter, what means content body will be ignored. Http with 404 code will produce just empty record.
+When server returns response with 500s code except 501, 505 and 506 then connector will re-send request based on configuration in `http.source.lookup.retry-strategy` parameters. By default it's fixed-delay with 1 second delay, up to 3 times per request (parameter `lookup.max-retries`). After exceeding max-retries limit the job will fail.
+A response with any other code than specified in params `success-codes` and `retry-codes` e.g. 400, 505, 301 will cause job failure.
+
+
+```roomsql
+CREATE TABLE [...]
+WITH (
+ [...],
+ 'http.source.lookup.success-codes' = '2XX',
+ 'http.source.lookup.retry-codes' = '',
+ 'http.source.lookup.ignored-response-codes' = '1XX,3XX,4XX,5XX'
+)
+```
+In this configuration, all HTTP responses are considered successful because the sets `success-codes` and `ignored-response-codes` together cover all possible status codes. As a result, no retries will be triggered based on HTTP response codes. However, only responses with status code 200 will be parsed and processed by the Flink operator. Responses with status codes in the 1xx, 3xx, 4xx, and 5xx ranges are classified under `ignored-response-codes`.
+Note that retries remain enabled and will still occur on IOException.
+To disable retries, set `'lookup.max-retries' = '0'`.
+
+## Security considerations
+
+### TLS (more secure replacement for SSL) and mTLS support
+
+Both Http Sink and Lookup Source connectors support HTTPS communication using TLS 1.2 and mTLS.
+To enable Https communication simply use `https` protocol in endpoint's URL.
+
+To specify certificate(s) to be used by the server, use `http.security.cert.server` connector property;
+the value is a comma separated list of paths to certificate(s), for example you can use your organization's CA
+Root certificate, or a self-signed certificate.
+
+Note that if there are no security properties for a `https` url then, the JVMs default certificates are
+used - allowing use of globally recognized CAs without the need for configuration.
+
+You can also configure the connector to use mTLS. For this simply use `http.security.cert.client`
+and `http.security.key.client` connector properties to specify paths to the certificate and
+private key. The key MUST be in `PCKS8` format. Both PEM and DER keys are
+allowed.
+
+For non production environments it is sometimes necessary to use Https connection and accept all certificates.
+In this special case, you can configure connector to trust all certificates without adding them to keystore.
+To enable this option use `http.security.cert.server.allowSelfSigned` property setting its value to `true`.
+
+### Basic Authentication
+The connector supports Basic Authentication using a HTTP `Authorization` header.
+The header value can be set via properties, similarly as for other headers. The connector converts the passed value to Base64 and uses it for the request.
+If the used value starts with the prefix `Basic`, or `http.source.lookup.use-raw-authorization-header`
+is set to `'true'`, it will be used as header value as is, without any extra modification.
+
+### OIDC Bearer Authentication
+The connector supports Bearer Authentication using a HTTP `Authorization` header. The [OAuth 2.0 rcf](https://datatracker.ietf.org/doc/html/rfc6749) mentions [Obtaining Authorization](https://datatracker.ietf.org/doc/html/rfc6749#section-4)
+and an authorization grant. OIDC makes use of this [authorisation grant](https://datatracker.ietf.org/doc/html/rfc6749#section-1.3) in a [Token Request](https://openid.net/specs/openid-connect-core-1_0.html#TokenRequest) by including a [OAuth grant type](https://oauth.net/2/grant-types/) and associated properties, the response is the [token response](https://openid.net/specs/openid-connect-core-1_0.html#TokenResponse).
+
+If you want to use this authorization then you should supply the `Token Request` body in `application/x-www-form-urlencoded` encoding
+in configuration property `http.security.oidc.token.request`. See [grant extension](https://datatracker.ietf.org/doc/html/rfc6749#section-4.5) for
+an example of a customised grant type token request. The supplied `token request` will be issued to the
+[token end point](https://datatracker.ietf.org/doc/html/rfc6749#section-3.2), whose url should be supplied in configuration property
+`http.security.oidc.token.endpoint.url`. The returned `access token` is then cached and used for subsequent requests; if the token has expired then
+a new one is requested. There is a property `http.security.oidc.token.expiry.reduction`, that defaults to 1 second; new tokens will
+be requested if the current time is later than the cached token expiry time minus `http.security.oidc.token.expiry.reduction`.
+
+#### Restrictions at this time
+* No authentication is applied to the token request.
+* The processing does not use the refresh token if it present.
+ {{< top >}}
diff --git a/flink-connector-http/pom.xml b/flink-connector-http/pom.xml
new file mode 100644
index 0000000..2951cc0
--- /dev/null
+++ b/flink-connector-http/pom.xml
@@ -0,0 +1,465 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-http-parent</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>flink-connector-http</artifactId>
+ <name>Flink : Connectors : http</name>
+ <version>1.0-SNAPSHOT</version>
+
+ <packaging>jar</packaging>
+
+ <properties>
+ <flink.connector.module.config><!--
+ FlinkKafkaProducerBaseTest --> --add-opens=java.base/java.lang=ALL-UNNAMED <!--
+ FlinkKafkaProducerBaseTest --> --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED <!--
+ FlinkKafkaConsumerBaseTest --> --add-opens=java.base/java.util=ALL-UNNAMED <!--
+ KafkaProducerExactlyOnceITCase -->
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ </flink.connector.module.config>
+ </properties>
+
+ <scm>
+ <url>https://github.com/apache/flink-connector-http</url>
+ <connection>git@github.com:apache/flink-connector-http.git</connection>
+ <developerConnection>
+ scm:git:https://gitbox.apache.org/repos/asf/flink-connector-http.git
+ </developerConnection>
+ </scm>
+
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <name>Apache Development Snapshot Repository</name>
+ <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <!-- Apache Flink dependencies -->
+ <!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-jackson</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-runtime</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-json</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-jackson</artifactId>
+ <version>${flink.shaded.jackson.version}-${flink.shaded.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Table ecosystem -->
+
+ <!-- Projects depending on this project won't depend on flink-table-*. -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-api-java-bridge</artifactId>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-api-java</artifactId>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>${lombok.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>1.3.9</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.vavr</groupId>
+ <artifactId>vavr</artifactId>
+ <version>0.10.2</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.github.resilience4j</groupId>
+ <artifactId>resilience4j-retry</artifactId>
+ <version>${resilence4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.github.resilience4j</groupId>
+ <artifactId>resilience4j-core</artifactId>
+ <version>${resilence4j.version}</version>
+ </dependency>
+
+ <!--http table factory testing -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-junit-jupiter</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-inline</artifactId>
+ <version>${mockito-inline.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-jackson</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- brings in junit 5.10.1 which mismatches our junit so we need to exclude it-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils-junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-jackson</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <version>${assertj.core.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <version>${junit5.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <version>${junit5.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>-org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ <version>2.5.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.github.tomakehurst</groupId>
+ <artifactId>wiremock</artifactId>
+ <version>${wiremock.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId> com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>32.0.1-jre</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>3.0.0-M5</version>
+ <configuration>
+ <!-- enables TCP/IP communication between surefire and forked JVM-->
+ <forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
+ <trimStackTrace>false</trimStackTrace>
+ <systemPropertyVariables>
+ <forkNumber>0${surefire.forkNumber}</forkNumber>
+ <!-- $$ ensures that surefire resolves this to the current forkNumber,
+ instead of maven during initialization -->
+ <mvn.forkNumber>$${surefire.forkNumber}</mvn.forkNumber>
+ <user.country>US</user.country>
+ <user.language>en</user.language>
+ <project.basedir>${project.basedir}</project.basedir>
+ <junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
+ </systemPropertyVariables>
+ <argLine>-Xms256m -Xmx2048m -XX:+UseG1GC</argLine>
+ </configuration>
+ <executions>
+ <execution>
+ <!--execute all the unit tests-->
+ <id>default-test</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>${test.unit.pattern}</include>
+ </includes>
+ <excludes>
+ <exclude>${additionalExcludes}</exclude>
+ </excludes>
+ <!--
+ The default forkCount from the parent pom is 4. This runs all the unit tests 4 times,
+ Each run of the unit tests as competing for the WireMockServer port, so we were seeing
+ errors where the port was already being used. forCount 1 means we run the unit tests once
+ and there is no port contention.
+ -->
+ <forkCount>1</forkCount>
+ <argLine>${flink.surefire.baseArgLine} -Xmx${flink.XmxUnitTest}</argLine>
+ </configuration>
+ </execution>
+
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <!-- activate API compatibility checks -->
+ <groupId>io.github.zentol.japicmp</groupId>
+ <artifactId>japicmp-maven-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>3.7.1</version>
+ <executions>
+ <execution>
+ <id>analyze-deps</id>
+ <goals>
+ <goal>analyze</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <failOnWarning>true</failOnWarning>
+ <ignoreAllNonTestScoped>true</ignoreAllNonTestScoped>
+ <ignoredDependencies>
+ <!--These dependencies are required for the unit tests to succeed -->
+ <ignoredDependency>org.mockito:mockito-inline:jar:5.2.0</ignoredDependency>
+ <ignoredDependency>net.bytebuddy:byte-buddy:jar:1.14.17</ignoredDependency>
+ <ignoredDependency>com.google.guava:guava:jar:32.0.1-jre</ignoredDependency>
+ <ignoredDependency>org.apache.flink:flink-clients:jar:1.20.0</ignoredDependency>
+ <ignoredUnusedDeclaredDependencies>org.apache.flink:flink-table-planner_${scala.binary.version}
+ </ignoredUnusedDeclaredDependencies>
+ <ignoredDependency>org.apache.flink:flink-table-test-utils:jar:1.20.0</ignoredDependency>
+ </ignoredDependencies>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-banned-deps</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <bannedDependencies>
+ <excludes>
+ <exclude>com.google.guava:guava</exclude>
+ <!--
+ TODO - need to remove Mockito from the unit test code
+ <exclude>org.mockito</exclude>
+ -->
+ <exclude>org.powermock</exclude>
+ </excludes>
+ <includes>
+ <include>com.google.guava:guava:*:*:test</include>
+ </includes>
+ </bannedDependencies>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallback.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallback.java
new file mode 100644
index 0000000..fc73ec9
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallback.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
+
+import java.io.Serializable;
+import java.net.http.HttpResponse;
+import java.util.Map;
+
+/**
+ * An interface for post request callback action, processing a response and its respective request.
+ *
+ * <p>One can customize the behaviour of such a callback by implementing both {@link
+ * HttpPostRequestCallback} and {@link HttpPostRequestCallbackFactory}.
+ *
+ * @param <RequestT> type of the HTTP request wrapper
+ */
+public interface HttpPostRequestCallback<RequestT> extends Serializable {
+ /**
+ * Process HTTP request and the matching response.
+ *
+ * @param response HTTP response
+ * @param requestEntry request's payload
+ * @param endpointUrl the URL of the endpoint
+ * @param headerMap mapping of header names to header values
+ */
+ void call(
+ HttpResponse<String> response,
+ RequestT requestEntry,
+ String endpointUrl,
+ Map<String, String> headerMap);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java
new file mode 100644
index 0000000..155053c
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpPostRequestCallbackFactory.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
+
+import org.apache.flink.connector.http.table.lookup.HttpLookupTableSource;
+import org.apache.flink.connector.http.table.sink.HttpDynamicSink;
+import org.apache.flink.table.factories.Factory;
+
+/**
+ * The {@link Factory} that dynamically creates and injects {@link HttpPostRequestCallback} to
+ * {@link HttpDynamicSink} and {@link HttpLookupTableSource}.
+ *
+ * <p>Custom implementations of {@link HttpPostRequestCallbackFactory} can be registered along other
+ * factories in
+ *
+ * <pre>resources/META-INF/services/org.apache.flink.table.factories.Factory</pre>
+ *
+ * <p>file and then referenced by their identifiers in:
+ *
+ * <ul>
+ * <li>The HttpSink DDL property field <i>http.sink.request-callback</i> for HTTP sink.
+ * <li>The Http lookup DDL property field <i>http.source.lookup.request-callback</i> for HTTP
+ * lookup.
+ * </ul>
+ *
+ * <br>
+ *
+ * <p>The following example shows the minimum Table API example to create a {@link HttpDynamicSink}
+ * that uses a custom callback created by a factory that returns <i>my-callback</i> as its
+ * identifier.
+ *
+ * <pre>{@code
+ * CREATE TABLE http (
+ * id bigint,
+ * some_field string
+ * ) with (
+ * 'connector' = 'http-sink'
+ * 'url' = 'http://example.com/myendpoint'
+ * 'format' = 'json',
+ * 'http.sink.request-callback' = 'my-callback'
+ * )
+ * }</pre>
+ *
+ * <p>The following example shows the minimum Table API example to create a {@link
+ * HttpLookupTableSource} that uses a custom callback created by a factory that returns
+ * <i>my-callback</i> as its identifier.
+ *
+ * <pre>{@code
+ * CREATE TABLE httplookup (
+ * id bigint
+ * ) with (
+ * 'connector' = 'rest-lookup',
+ * 'url' = 'http://example.com/myendpoint',
+ * 'format' = 'json',
+ * 'http.source.lookup.request-callback' = 'my-callback'
+ * )
+ * }</pre>
+ *
+ * @param <RequestT> type of the HTTP request wrapper
+ */
+public interface HttpPostRequestCallbackFactory<RequestT> extends Factory {
+ /** @return {@link HttpPostRequestCallback} custom request callback instance */
+ HttpPostRequestCallback<RequestT> createHttpPostRequestCallback();
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSink.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSink.java
new file mode 100644
index 0000000..3afbee4
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSink.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.connector.base.sink.writer.ElementConverter;
+import org.apache.flink.connector.http.clients.SinkHttpClientBuilder;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.sink.HttpSinkInternal;
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+import org.apache.flink.connector.http.sink.httpclient.HttpRequest;
+
+import java.util.Properties;
+
+/**
+ * A public implementation for {@code HttpSink} that performs async requests against a specified
+ * HTTP endpoint using the buffering protocol specified in {@link
+ * org.apache.flink.connector.base.sink.AsyncSinkBase}.
+ *
+ * <p>To create a new instance of this class use {@link HttpSinkBuilder}. An example would be:
+ *
+ * <pre>{@code
+ * HttpSink<String> httpSink =
+ * HttpSink.<String>builder()
+ * .setEndpointUrl("http://example.com/myendpoint")
+ * .setElementConverter(
+ * (s, _context) -> new HttpSinkRequestEntry("POST", "text/plain",
+ * s.getBytes(StandardCharsets.UTF_8)))
+ * .build();
+ * }</pre>
+ *
+ * @param <InputT> type of the elements that should be sent through HTTP request.
+ */
+@PublicEvolving
+public class HttpSink<InputT> extends HttpSinkInternal<InputT> {
+
+ HttpSink(
+ ElementConverter<InputT, HttpSinkRequestEntry> elementConverter,
+ int maxBatchSize,
+ int maxInFlightRequests,
+ int maxBufferedRequests,
+ long maxBatchSizeInBytes,
+ long maxTimeInBufferMS,
+ long maxRecordSizeInBytes,
+ String endpointUrl,
+ HttpPostRequestCallback<HttpRequest> httpPostRequestCallback,
+ HeaderPreprocessor headerPreprocessor,
+ SinkHttpClientBuilder sinkHttpClientBuilder,
+ Properties properties) {
+
+ super(
+ elementConverter,
+ maxBatchSize,
+ maxInFlightRequests,
+ maxBufferedRequests,
+ maxBatchSizeInBytes,
+ maxTimeInBufferMS,
+ maxRecordSizeInBytes,
+ endpointUrl,
+ httpPostRequestCallback,
+ headerPreprocessor,
+ sinkHttpClientBuilder,
+ properties);
+ }
+
+ /**
+ * Create a {@link HttpSinkBuilder} constructing a new {@link HttpSink}.
+ *
+ * @param <InputT> type of the elements that should be sent through HTTP request
+ * @return {@link HttpSinkBuilder}
+ */
+ public static <InputT> HttpSinkBuilder<InputT> builder() {
+ return new HttpSinkBuilder<>();
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/HttpSinkBuilder.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSinkBuilder.java
similarity index 60%
rename from src/main/java/com/getindata/connectors/http/HttpSinkBuilder.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSinkBuilder.java
index 208bcf0..16b6e8c 100644
--- a/src/main/java/com/getindata/connectors/http/HttpSinkBuilder.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpSinkBuilder.java
@@ -1,20 +1,36 @@
-package com.getindata.connectors.http;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.util.Optional;
-import java.util.Properties;
+package org.apache.flink.connector.http;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.connector.base.sink.AsyncSinkBaseBuilder;
import org.apache.flink.connector.base.sink.writer.ElementConverter;
+import org.apache.flink.connector.http.clients.SinkHttpClient;
+import org.apache.flink.connector.http.clients.SinkHttpClientBuilder;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+import org.apache.flink.connector.http.sink.httpclient.HttpRequest;
+import org.apache.flink.connector.http.sink.httpclient.JavaNetSinkHttpClient;
+import org.apache.flink.connector.http.table.sink.Slf4jHttpPostRequestCallback;
+import org.apache.flink.connector.http.utils.HttpHeaderUtils;
-import com.getindata.connectors.http.internal.HeaderPreprocessor;
-import com.getindata.connectors.http.internal.SinkHttpClient;
-import com.getindata.connectors.http.internal.SinkHttpClientBuilder;
-import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry;
-import com.getindata.connectors.http.internal.sink.httpclient.HttpRequest;
-import com.getindata.connectors.http.internal.sink.httpclient.JavaNetSinkHttpClient;
-import com.getindata.connectors.http.internal.table.sink.Slf4jHttpPostRequestCallback;
-import com.getindata.connectors.http.internal.utils.HttpHeaderUtils;
+import java.util.Optional;
+import java.util.Properties;
/**
* Builder to construct {@link HttpSink}.
@@ -34,20 +50,22 @@ import com.getindata.connectors.http.internal.utils.HttpHeaderUtils;
* }</pre>
*
* <p>If the following parameters are not set in this builder, the following defaults will be used:
+ *
* <ul>
- * <li>{@code maxBatchSize} will be 500,</li>
- * <li>{@code maxInFlightRequests} will be 50,</li>
- * <li>{@code maxBufferedRequests} will be 10000,</li>
- * <li>{@code maxBatchSizeInBytes} will be 5 MB i.e. {@code 5 * 1024 * 1024},</li>
- * <li>{@code maxTimeInBufferMS} will be 5000ms,</li>
- * <li>{@code maxRecordSizeInBytes} will be 1 MB i.e. {@code 1024 * 1024}.</li>
+ * <li>{@code maxBatchSize} will be 500,
+ * <li>{@code maxInFlightRequests} will be 50,
+ * <li>{@code maxBufferedRequests} will be 10000,
+ * <li>{@code maxBatchSizeInBytes} will be 5 MB i.e. {@code 5 * 1024 * 1024},
+ * <li>{@code maxTimeInBufferMS} will be 5000ms,
+ * <li>{@code maxRecordSizeInBytes} will be 1 MB i.e. {@code 1024 * 1024}.
* </ul>
+ *
* {@code endpointUrl} and {@code elementConverter} must be set by the user.
*
* @param <InputT> type of the elements that should be sent through HTTP request.
*/
-public class HttpSinkBuilder<InputT> extends
- AsyncSinkBaseBuilder<InputT, HttpSinkRequestEntry, HttpSinkBuilder<InputT>> {
+public class HttpSinkBuilder<InputT>
+ extends AsyncSinkBaseBuilder<InputT, HttpSinkRequestEntry, HttpSinkBuilder<InputT>> {
private static final int DEFAULT_MAX_BATCH_SIZE = 500;
@@ -63,11 +81,11 @@ public class HttpSinkBuilder<InputT> extends
private static final SinkHttpClientBuilder DEFAULT_CLIENT_BUILDER = JavaNetSinkHttpClient::new;
- private static final HttpPostRequestCallback<HttpRequest>
- DEFAULT_POST_REQUEST_CALLBACK = new Slf4jHttpPostRequestCallback();
+ private static final HttpPostRequestCallback<HttpRequest> DEFAULT_POST_REQUEST_CALLBACK =
+ new Slf4jHttpPostRequestCallback();
private static final HeaderPreprocessor DEFAULT_HEADER_PREPROCESSOR =
- HttpHeaderUtils.createBasicAuthorizationHeaderPreprocessor();
+ HttpHeaderUtils.createBasicAuthorizationHeaderPreprocessor();
private final Properties properties = new Properties();
@@ -103,11 +121,11 @@ public class HttpSinkBuilder<InputT> extends
/**
* @param sinkHttpClientBuilder builder for an implementation of {@link SinkHttpClient} that
- * will be used by {@link HttpSink}
+ * will be used by {@link HttpSink}
* @return {@link HttpSinkBuilder} itself
*/
public HttpSinkBuilder<InputT> setSinkHttpClientBuilder(
- SinkHttpClientBuilder sinkHttpClientBuilder) {
+ SinkHttpClientBuilder sinkHttpClientBuilder) {
this.sinkHttpClientBuilder = sinkHttpClientBuilder;
return this;
}
@@ -116,30 +134,30 @@ public class HttpSinkBuilder<InputT> extends
* @param elementConverter the {@link ElementConverter} to be used for the sink
* @return {@link HttpSinkBuilder} itself
* @deprecated Converters set by this method might not work properly for Flink 1.16+. Use {@link
- * #setElementConverter(SchemaLifecycleAwareElementConverter)} instead.
+ * #setElementConverter(SchemaLifecycleAwareElementConverter)} instead.
*/
@Deprecated
@PublicEvolving
public HttpSinkBuilder<InputT> setElementConverter(
- ElementConverter<InputT, HttpSinkRequestEntry> elementConverter) {
+ ElementConverter<InputT, HttpSinkRequestEntry> elementConverter) {
this.elementConverter = elementConverter;
return this;
}
/**
* @param elementConverter the {@link SchemaLifecycleAwareElementConverter} to be used for the
- * sink
+ * sink
* @return {@link HttpSinkBuilder} itself
*/
@PublicEvolving
public HttpSinkBuilder<InputT> setElementConverter(
- SchemaLifecycleAwareElementConverter<InputT, HttpSinkRequestEntry> elementConverter) {
+ SchemaLifecycleAwareElementConverter<InputT, HttpSinkRequestEntry> elementConverter) {
this.elementConverter = elementConverter;
return this;
}
public HttpSinkBuilder<InputT> setHttpPostRequestCallback(
- HttpPostRequestCallback<HttpRequest> httpPostRequestCallback) {
+ HttpPostRequestCallback<HttpRequest> httpPostRequestCallback) {
this.httpPostRequestCallback = httpPostRequestCallback;
return this;
}
@@ -152,6 +170,7 @@ public class HttpSinkBuilder<InputT> extends
/**
* Set property for Http Sink.
+ *
* @param propertyName property name
* @param propertyValue property value
* @return {@link HttpSinkBuilder} itself
@@ -162,7 +181,8 @@ public class HttpSinkBuilder<InputT> extends
}
/**
- * Add properties to Http Sink configuration
+ * Add properties to Http Sink configuration.
+ *
* @param properties properties to add
* @return {@link HttpSinkBuilder} itself
*/
@@ -174,18 +194,18 @@ public class HttpSinkBuilder<InputT> extends
@Override
public HttpSink<InputT> build() {
return new HttpSink<>(
- elementConverter,
- Optional.ofNullable(getMaxBatchSize()).orElse(DEFAULT_MAX_BATCH_SIZE),
- Optional.ofNullable(getMaxInFlightRequests()).orElse(DEFAULT_MAX_IN_FLIGHT_REQUESTS),
- Optional.ofNullable(getMaxBufferedRequests()).orElse(DEFAULT_MAX_BUFFERED_REQUESTS),
- Optional.ofNullable(getMaxBatchSizeInBytes()).orElse(DEFAULT_MAX_BATCH_SIZE_IN_B),
- Optional.ofNullable(getMaxTimeInBufferMS()).orElse(DEFAULT_MAX_TIME_IN_BUFFER_MS),
- Optional.ofNullable(getMaxRecordSizeInBytes()).orElse(DEFAULT_MAX_RECORD_SIZE_IN_B),
- endpointUrl,
- httpPostRequestCallback,
- headerPreprocessor,
- sinkHttpClientBuilder,
- properties
- );
+ elementConverter,
+ Optional.ofNullable(getMaxBatchSize()).orElse(DEFAULT_MAX_BATCH_SIZE),
+ Optional.ofNullable(getMaxInFlightRequests())
+ .orElse(DEFAULT_MAX_IN_FLIGHT_REQUESTS),
+ Optional.ofNullable(getMaxBufferedRequests()).orElse(DEFAULT_MAX_BUFFERED_REQUESTS),
+ Optional.ofNullable(getMaxBatchSizeInBytes()).orElse(DEFAULT_MAX_BATCH_SIZE_IN_B),
+ Optional.ofNullable(getMaxTimeInBufferMS()).orElse(DEFAULT_MAX_TIME_IN_BUFFER_MS),
+ Optional.ofNullable(getMaxRecordSizeInBytes()).orElse(DEFAULT_MAX_RECORD_SIZE_IN_B),
+ endpointUrl,
+ httpPostRequestCallback,
+ headerPreprocessor,
+ sinkHttpClientBuilder,
+ properties);
}
}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpStatusCodeValidationFailedException.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpStatusCodeValidationFailedException.java
new file mode 100644
index 0000000..b1acd45
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/HttpStatusCodeValidationFailedException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
+
+import lombok.Getter;
+
+import java.net.http.HttpResponse;
+
+/** Exception passing back a message and http response for error processing. */
+@Getter
+public class HttpStatusCodeValidationFailedException extends Exception {
+ private final HttpResponse<?> response;
+
+ public HttpStatusCodeValidationFailedException(String message, HttpResponse<?> response) {
+ super(message);
+ this.response = response;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupArg.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupArg.java
new file mode 100644
index 0000000..b5a584d
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupArg.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
+
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+/** Transfer object that contains single lookup argument (column name) and its value. */
+@Data
+@RequiredArgsConstructor
+public class LookupArg {
+
+ /** Lookup argument name. */
+ private final String argName;
+
+ /** Lookup argument value. */
+ private final String argValue;
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreator.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreator.java
new file mode 100644
index 0000000..e31fe13
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreator.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
+
+import org.apache.flink.connector.http.table.lookup.LookupQueryInfo;
+import org.apache.flink.table.data.RowData;
+
+import java.io.Serializable;
+
+/**
+ * An interface for a creator of a lookup query in the Http Lookup Source (e.g., the query that gets
+ * appended as query parameters to the URI in GET request or supplied as the payload of a body-based
+ * request along with optional query parameters).
+ *
+ * <p>One can customize how those queries are built by implementing {@link LookupQueryCreator} and
+ * {@link LookupQueryCreatorFactory}.
+ */
+public interface LookupQueryCreator extends Serializable {
+
+ /**
+ * Create a lookup query (like the query appended to path in GET request) out of the provided
+ * arguments.
+ *
+ * @param lookupDataRow a {@link RowData} containing request parameters.
+ * @return a lookup query.
+ */
+ LookupQueryInfo createLookupQuery(RowData lookupDataRow);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreatorFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreatorFactory.java
new file mode 100644
index 0000000..d3a6322
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/LookupQueryCreatorFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
+
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.http.table.lookup.HttpLookupTableSource;
+import org.apache.flink.connector.http.table.lookup.LookupRow;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.Factory;
+
+import java.io.Serializable;
+
+/**
+ * The {@link Factory} that dynamically creates and injects {@link LookupQueryCreator} to {@link
+ * HttpLookupTableSource}.
+ *
+ * <p>Custom implementations of {@link LookupQueryCreatorFactory} can be registered along other
+ * factories in
+ *
+ * <pre>resources/META-INF.services/org.apache.flink.table.factories.Factory</pre>
+ *
+ * <p>file and then referenced by their identifiers in the HttpLookupSource DDL property field
+ * <i>flink.connector.http.source.lookup.query-creator</i>.
+ *
+ * <p>The following example shows the minimum Table API example to create a {@link
+ * HttpLookupTableSource} that uses a custom query creator created by a factory that returns
+ * <i>my-query-creator</i> as its identifier.
+ *
+ * <pre>{@code
+ * CREATE TABLE http (
+ * id bigint,
+ * some_field string
+ * ) WITH (
+ * 'connector' = 'rest-lookup',
+ * 'format' = 'json',
+ * 'url' = 'http://example.com/myendpoint',
+ * 'http.source.lookup.query-creator' = 'my-query-creator'
+ * )
+ * }</pre>
+ */
+public interface LookupQueryCreatorFactory extends Factory, Serializable {
+
+ /**
+ * @param readableConfig readable config
+ * @param lookupRow lookup row
+ * @param dynamicTableFactoryContext context
+ * @return {@link LookupQueryCreator} custom lookup query creator instance
+ */
+ LookupQueryCreator createLookupQueryCreator(
+ ReadableConfig readableConfig,
+ LookupRow lookupRow,
+ DynamicTableFactory.Context dynamicTableFactoryContext);
+}
diff --git a/src/main/java/com/getindata/connectors/http/SchemaLifecycleAwareElementConverter.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/SchemaLifecycleAwareElementConverter.java
similarity index 52%
rename from src/main/java/com/getindata/connectors/http/SchemaLifecycleAwareElementConverter.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/SchemaLifecycleAwareElementConverter.java
index c4986ba..9849a0e 100644
--- a/src/main/java/com/getindata/connectors/http/SchemaLifecycleAwareElementConverter.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/SchemaLifecycleAwareElementConverter.java
@@ -1,4 +1,21 @@
-package com.getindata.connectors.http;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http;
import org.apache.flink.api.common.serialization.SerializationSchema.InitializationContext;
import org.apache.flink.api.connector.sink2.Sink.InitContext;
@@ -9,9 +26,9 @@ import org.apache.flink.connector.base.sink.writer.ElementConverter;
* that will be called by HTTP connect code to ensure that element converter is initialized
* properly. This is required for cases when Flink's SerializationSchema and DeserializationSchema
* objects like JsonRowDataSerializationSchema are used.
- * <p>
- * This interface specifies the mapping between elements of a stream to request entries that can be
- * sent to the destination. The mapping is provided by the end-user of a sink, not the sink
+ *
+ * <p>This interface specifies the mapping between elements of a stream to request entries that can
+ * be sent to the destination. The mapping is provided by the end-user of a sink, not the sink
* creator.
*
* <p>The request entries contain all relevant information required to create and sent the actual
@@ -19,7 +36,7 @@ import org.apache.flink.connector.base.sink.writer.ElementConverter;
* key.
*/
public interface SchemaLifecycleAwareElementConverter<InputT, RequestEntryT>
- extends ElementConverter<InputT, RequestEntryT> {
+ extends ElementConverter<InputT, RequestEntryT> {
/**
* Initialization element converter for the schema.
@@ -30,5 +47,4 @@ public interface SchemaLifecycleAwareElementConverter<InputT, RequestEntryT>
* @param context Contextual information that can be used during initialization.
*/
void open(InitContext context);
-
}
diff --git a/src/main/java/com/getindata/connectors/http/internal/auth/OidcAccessTokenManager.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/auth/OidcAccessTokenManager.java
similarity index 66%
rename from src/main/java/com/getindata/connectors/http/internal/auth/OidcAccessTokenManager.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/auth/OidcAccessTokenManager.java
index 9334870..7145803 100644
--- a/src/main/java/com/getindata/connectors/http/internal/auth/OidcAccessTokenManager.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/auth/OidcAccessTokenManager.java
@@ -1,12 +1,12 @@
-
/*
- * Copyright 2020 Red Hat
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,7 +15,13 @@
* limitations under the License.
*/
-package com.getindata.connectors.http.internal.auth;
+package org.apache.flink.connector.http.auth;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.net.URI;
@@ -25,20 +31,14 @@ import java.net.http.HttpResponse;
import java.time.Duration;
import java.time.Instant;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-
/**
- * This class is inspired by
- * https://github.com/Apicurio/apicurio-common-rest-client/blob/
+ * This class is inspired by https://github.com/Apicurio/apicurio-common-rest-client/blob/
* 944ac9eb527c291a6083bd10ee012388e1684d20/rest-client-common/src/main/java/io/
* apicurio/rest/client/auth/OidcAuth.java.
*
- * The OIDC access token manager encapsulates the caching of an OIDC access token,
- * which can be short lived, for example an hour. The authenticate method will return an
- * un-expired access token, either from the cache or by requesting a new access token.
+ * <p>The OIDC access token manager encapsulates the caching of an OIDC access token, which can be
+ * short lived, for example an hour. The authenticate method will return an un-expired access token,
+ * either from the cache or by requesting a new access token.
*/
@Slf4j
public class OidcAccessTokenManager {
@@ -54,7 +54,8 @@ public class OidcAccessTokenManager {
private Instant cachedAccessTokenExp;
/**
- * Construct an Oidc access token manager with the default token expiration reduction
+ * Construct an Oidc access token manager with the default token expiration reduction.
+ *
* @param httpClient httpClient to use to call the token endpoint.
* @param tokenRequest token request
* @param url token endpoint url
@@ -62,19 +63,22 @@ public class OidcAccessTokenManager {
public OidcAccessTokenManager(HttpClient httpClient, String tokenRequest, String url) {
this(httpClient, tokenRequest, url, DEFAULT_TOKEN_EXPIRATION_REDUCTION);
}
+
/**
- * Construct an Oidc access token manager with the supplied token expiration reduction
+ * Construct an Oidc access token manager with the supplied token expiration reduction.
+ *
* @param httpClient httpClient to use to call the token endpoint.
* @param tokenRequest token request this need to be form urlencoded
* @param url token endpoint url
- * @param tokenExpirationReduction token expiry reduction, request a new token if the
- * current time is later than the cached access token
- * expiry time reduced by this value. This means that
- * we will not use the cached token if it is about
- * to expire.
+ * @param tokenExpirationReduction token expiry reduction, request a new token if the current
+ * time is later than the cached access token expiry time reduced by this value. This means
+ * that we will not use the cached token if it is about to expire.
*/
- public OidcAccessTokenManager(HttpClient httpClient, String tokenRequest, String url,
- Duration tokenExpirationReduction) {
+ public OidcAccessTokenManager(
+ HttpClient httpClient,
+ String tokenRequest,
+ String url,
+ Duration tokenExpirationReduction) {
this.tokenRequest = tokenRequest;
this.httpClient = httpClient;
this.url = url;
@@ -85,21 +89,19 @@ public class OidcAccessTokenManager {
}
}
- /**
- * Request an access token from the token endpoint
- */
+ /** Request an access token from the token endpoint. */
private void requestAccessToken() {
try {
HttpRequest httpRequest =
HttpRequest.newBuilder()
- .uri(URI.create(url))
- .header("Content-Type", "application/x-www-form-urlencoded")
- .method("POST", HttpRequest.BodyPublishers.ofString(tokenRequest))
- .build();
+ .uri(URI.create(url))
+ .header("Content-Type", "application/x-www-form-urlencoded")
+ .method("POST", HttpRequest.BodyPublishers.ofString(tokenRequest))
+ .build();
- HttpResponse<byte[]> response = httpClient.send(httpRequest,
- HttpResponse.BodyHandlers.ofByteArray());
- //create ObjectMapper instance
+ HttpResponse<byte[]> response =
+ httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray());
+ // create ObjectMapper instance
final ObjectMapper objectMapper = new ObjectMapper();
if (200 == response.statusCode()) {
byte[] bytes = response.body();
@@ -112,18 +114,20 @@ public class OidcAccessTokenManager {
*/
Duration expiresIn = Duration.ofSeconds(expiresInNode.asInt());
if (expiresIn.compareTo(this.tokenExpirationReduction) > 0) {
- //expiresIn is greater than tokenExpirationReduction
+ // expiresIn is greater than tokenExpirationReduction
expiresIn = expiresIn.minus(this.tokenExpirationReduction);
}
this.cachedAccessTokenExp = Instant.now().plus(expiresIn);
} else {
- throw new IllegalStateException("Attempted to get an access token but got http" +
- " status code " + response.statusCode());
+ throw new IllegalStateException(
+ "Attempted to get an access token but got http"
+ + " status code "
+ + response.statusCode());
}
} catch (JsonProcessingException e) {
throw new IllegalStateException("Error found while trying to request a new token");
} catch (IOException e) {
- throw new IllegalStateException("IO Exception occurred", e);
+ throw new IllegalStateException("IO Exception occurred", e);
} catch (InterruptedException e) {
throw new IllegalStateException("Interrupted Exception occurred", e);
}
@@ -131,6 +135,7 @@ public class OidcAccessTokenManager {
/**
* Get a valid unexpired access token.
+ *
* @return access token.
*/
public String authenticate() {
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClient.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClient.java
new file mode 100644
index 0000000..6aa9703
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClient.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.clients;
+
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.FunctionContext;
+
+import java.util.Collection;
+
+/** A client that is used to get enrichment data from external component. */
+public interface PollingClient<T> {
+
+ /**
+ * Gets enrichment data from external component using provided lookup arguments.
+ *
+ * @param lookupRow A {@link RowData} containing request parameters.
+ * @return an optional result of data lookup.
+ */
+ Collection<T> pull(RowData lookupRow);
+
+ /**
+ * Initialize the client.
+ *
+ * @param ctx function context
+ */
+ void open(FunctionContext ctx);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClientFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClientFactory.java
new file mode 100644
index 0000000..4874e60
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/PollingClientFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.clients;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.connector.http.table.lookup.HttpLookupConfig;
+import org.apache.flink.util.ConfigurationException;
+
+import java.io.Serializable;
+
+/**
+ * Polling client factory.
+ *
+ * @param <OUT> polling client
+ */
+public interface PollingClientFactory<OUT> extends Serializable {
+
+ PollingClient<OUT> createPollClient(
+ HttpLookupConfig options, DeserializationSchema<OUT> schemaDecoder)
+ throws ConfigurationException;
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClient.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClient.java
new file mode 100644
index 0000000..ce9fc5b
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClient.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.clients;
+
+import org.apache.flink.connector.http.sink.HttpSinkInternal;
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+import org.apache.flink.connector.http.sink.HttpSinkWriter;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * An HTTP client that is used by {@link HttpSinkWriter} to send HTTP requests processed by {@link
+ * HttpSinkInternal}.
+ */
+public interface SinkHttpClient {
+
+ /**
+ * Sends HTTP requests to an external web service.
+ *
+ * @param requestEntries a set of request entries that should be sent to the destination
+ * @param endpointUrl the URL of the endpoint
+ * @return the new {@link CompletableFuture} wrapping {@link SinkHttpClientResponse} that
+ * completes when all requests have been sent and returned their statuses
+ */
+ CompletableFuture<SinkHttpClientResponse> putRequests(
+ List<HttpSinkRequestEntry> requestEntries, String endpointUrl);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientBuilder.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientBuilder.java
new file mode 100644
index 0000000..26352c3
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientBuilder.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.clients;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.connector.http.HttpPostRequestCallback;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.sink.httpclient.HttpRequest;
+import org.apache.flink.connector.http.sink.httpclient.RequestSubmitterFactory;
+
+import java.io.Serializable;
+import java.util.Properties;
+
+/** Builder building {@link SinkHttpClient}. */
+@PublicEvolving
+public interface SinkHttpClientBuilder extends Serializable {
+
+ // TODO Consider moving HttpPostRequestCallback and HeaderPreprocessor, RequestSubmitter to be a
+ // SinkHttpClientBuilder fields. This method is getting more and more arguments.
+ SinkHttpClient build(
+ Properties properties,
+ HttpPostRequestCallback<HttpRequest> httpPostRequestCallback,
+ HeaderPreprocessor headerPreprocessor,
+ RequestSubmitterFactory requestSubmitterFactory);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientResponse.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientResponse.java
new file mode 100644
index 0000000..9fbf169
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/clients/SinkHttpClientResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.clients;
+
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+import org.apache.flink.connector.http.sink.httpclient.HttpRequest;
+
+import lombok.Data;
+import lombok.NonNull;
+import lombok.ToString;
+
+import java.util.List;
+
+/**
+ * Data class holding {@link HttpSinkRequestEntry} instances that {@link SinkHttpClient} attempted
+ * to write, divided into two lists — successful and failed ones.
+ */
+@Data
+@ToString
+public class SinkHttpClientResponse {
+
+ /** A list of successfully written requests. */
+ @NonNull private final List<HttpRequest> successfulRequests;
+
+ /** A list of requests that {@link SinkHttpClient} failed to write. */
+ @NonNull private final List<HttpRequest> failedRequests;
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/ConfigException.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/ConfigException.java
new file mode 100644
index 0000000..cd296a7
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/ConfigException.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.config;
+
+/**
+ * A Runtime exception throw when there is any issue with configuration properties for Http
+ * Connector.
+ */
+public class ConfigException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public ConfigException(String message) {
+ super(message);
+ }
+
+ public ConfigException(String message, Throwable t) {
+ super(message, t);
+ }
+
+ /**
+ * Creates an exception object using predefined exception message template: {@code Invalid value
+ * + (value) + for configuration + (property name) + (additional message) }.
+ *
+ * @param name configuration property name.
+ * @param value configuration property value.
+ * @param message custom message appended to the end of exception message.
+ */
+ public ConfigException(String name, Object value, String message) {
+ super(
+ "Invalid value "
+ + value
+ + " for configuration "
+ + name
+ + (message == null ? "" : ": " + message));
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/HttpConnectorConfigConstants.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/HttpConnectorConfigConstants.java
new file mode 100644
index 0000000..9f7721a
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/HttpConnectorConfigConstants.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.config;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import lombok.experimental.UtilityClass;
+
+/** A dictionary class containing properties or properties prefixes for Http connector. */
+@UtilityClass
+@NoArgsConstructor(access = AccessLevel.NONE)
+// TODO Change this name to HttpConnectorConfigProperties
+public final class HttpConnectorConfigConstants {
+
+ public static final String PROP_DELIM = ",";
+
+ /** A property prefix for http connector. */
+ public static final String FLINK_CONNECTOR_HTTP = "http.";
+
+ private static final String SOURCE_LOOKUP_PREFIX = FLINK_CONNECTOR_HTTP + "source.lookup.";
+
+ /** A property prefix for http connector header properties. */
+ public static final String SINK_HEADER_PREFIX = FLINK_CONNECTOR_HTTP + "sink.header.";
+
+ public static final String LOOKUP_SOURCE_HEADER_PREFIX = SOURCE_LOOKUP_PREFIX + "header.";
+ public static final String OIDC_AUTH_TOKEN_REQUEST =
+ FLINK_CONNECTOR_HTTP + "security.oidc.token.request";
+
+ public static final String OIDC_AUTH_TOKEN_ENDPOINT_URL =
+ FLINK_CONNECTOR_HTTP + "security.oidc.token.endpoint.url";
+
+ public static final String OIDC_AUTH_TOKEN_EXPIRY_REDUCTION =
+ FLINK_CONNECTOR_HTTP + "security.oidc.token.expiry.reduction";
+ /**
+ * Whether to use the raw value of the Authorization header. If set, it prevents the special
+ * treatment of the header for Basic Authentication, thus preserving the passed raw value.
+ * Defaults to false.
+ */
+ public static final String LOOKUP_SOURCE_HEADER_USE_RAW =
+ SOURCE_LOOKUP_PREFIX + "use-raw-authorization-header";
+
+ public static final String RESULT_TYPE = SOURCE_LOOKUP_PREFIX + "result-type";
+
+ // --------- Error code handling configuration ---------
+
+ // TODO copied from
+ // https://github.com/getindata/flink-http-connector/blob/e00d57607f7d1a0d72c6ca48abe[…]nnectors/http/internal/config/HttpConnectorConfigConstants.java
+ // Changing label name to INCLUDE, but the value is exclude. Needs investigating.
+ public static final String HTTP_ERROR_SINK_CODE_INCLUDE_LIST =
+ FLINK_CONNECTOR_HTTP + "sink.error.code.exclude";
+
+ public static final String HTTP_ERROR_SINK_CODES_LIST =
+ FLINK_CONNECTOR_HTTP + "sink.error.code";
+ // -----------------------------------------------------
+
+ public static final String SOURCE_LOOKUP_REQUEST_CALLBACK_IDENTIFIER =
+ SOURCE_LOOKUP_PREFIX + "request-callback";
+
+ public static final String SINK_REQUEST_CALLBACK_IDENTIFIER =
+ FLINK_CONNECTOR_HTTP + "sink.request-callback";
+
+ public static final String SOURCE_LOOKUP_QUERY_CREATOR_IDENTIFIER =
+ SOURCE_LOOKUP_PREFIX + "query-creator";
+
+ // -------------- HTTPS security settings --------------
+ public static final String ALLOW_SELF_SIGNED =
+ FLINK_CONNECTOR_HTTP + "security.cert.server.allowSelfSigned";
+
+ public static final String SERVER_TRUSTED_CERT = FLINK_CONNECTOR_HTTP + "security.cert.server";
+
+ public static final String CLIENT_CERT = FLINK_CONNECTOR_HTTP + "security.cert.client";
+
+ public static final String CLIENT_PRIVATE_KEY = FLINK_CONNECTOR_HTTP + "security.key.client";
+
+ public static final String KEY_STORE_PATH = FLINK_CONNECTOR_HTTP + "security.keystore.path";
+
+ public static final String KEY_STORE_PASSWORD =
+ FLINK_CONNECTOR_HTTP + "security.keystore.password";
+
+ public static final String KEY_STORE_TYPE = FLINK_CONNECTOR_HTTP + "security.keystore.type";
+
+ // -----------------------------------------------------
+
+ // ------ HTTPS timeouts and thread pool settings ------
+
+ public static final String LOOKUP_HTTP_TIMEOUT_SECONDS =
+ SOURCE_LOOKUP_PREFIX + "request.timeout";
+
+ public static final String SOURCE_CONNECTION_TIMEOUT =
+ SOURCE_LOOKUP_PREFIX + "connection.timeout";
+
+ public static final String SOURCE_PROXY_HOST = SOURCE_LOOKUP_PREFIX + "proxy.host";
+
+ public static final String SOURCE_PROXY_PORT = SOURCE_LOOKUP_PREFIX + "proxy.port";
+
+ public static final String SOURCE_PROXY_USERNAME = SOURCE_LOOKUP_PREFIX + "proxy.username";
+
+ public static final String SOURCE_PROXY_PASSWORD = SOURCE_LOOKUP_PREFIX + "proxy.password";
+
+ public static final String SINK_HTTP_TIMEOUT_SECONDS =
+ FLINK_CONNECTOR_HTTP + "sink.request.timeout";
+
+ public static final String LOOKUP_HTTP_PULING_THREAD_POOL_SIZE =
+ SOURCE_LOOKUP_PREFIX + "request.thread-pool.size";
+
+ public static final String LOOKUP_HTTP_RESPONSE_THREAD_POOL_SIZE =
+ SOURCE_LOOKUP_PREFIX + "response.thread-pool.size";
+
+ public static final String SINK_HTTP_WRITER_THREAD_POOL_SIZE =
+ FLINK_CONNECTOR_HTTP + "sink.writer.thread-pool.size";
+
+ // -----------------------------------------------------
+
+ // ------ Sink request submitter settings ------
+ public static final String SINK_HTTP_REQUEST_MODE =
+ FLINK_CONNECTOR_HTTP + "sink.writer.request.mode";
+
+ public static final String SINK_HTTP_BATCH_REQUEST_SIZE =
+ FLINK_CONNECTOR_HTTP + "sink.request.batch.size";
+
+ // ---------------------------------------------
+ public static final String SOURCE_RETRY_SUCCESS_CODES = SOURCE_LOOKUP_PREFIX + "success-codes";
+ public static final String SOURCE_RETRY_RETRY_CODES = SOURCE_LOOKUP_PREFIX + "retry-codes";
+ public static final String SOURCE_IGNORE_RESPONSE_CODES =
+ SOURCE_LOOKUP_PREFIX + "ignored-response-codes";
+
+ public static final String SOURCE_RETRY_STRATEGY_PREFIX =
+ SOURCE_LOOKUP_PREFIX + "retry-strategy.";
+ public static final String SOURCE_RETRY_STRATEGY_TYPE = SOURCE_RETRY_STRATEGY_PREFIX + "type";
+
+ private static final String SOURCE_RETRY_FIXED_DELAY_PREFIX =
+ SOURCE_RETRY_STRATEGY_PREFIX + "fixed-delay.";
+ public static final String SOURCE_RETRY_FIXED_DELAY_DELAY =
+ SOURCE_RETRY_FIXED_DELAY_PREFIX + "delay";
+
+ private static final String SOURCE_RETRY_EXP_DELAY_PREFIX =
+ SOURCE_RETRY_STRATEGY_PREFIX + "exponential-delay.";
+ public static final String SOURCE_RETRY_EXP_DELAY_INITIAL_BACKOFF =
+ SOURCE_RETRY_EXP_DELAY_PREFIX + "initial-backoff";
+ public static final String SOURCE_RETRY_EXP_DELAY_MAX_BACKOFF =
+ SOURCE_RETRY_EXP_DELAY_PREFIX + "max-backoff";
+ public static final String SOURCE_RETRY_EXP_DELAY_MULTIPLIER =
+ SOURCE_RETRY_EXP_DELAY_PREFIX + "backoff-multiplier";
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/SinkRequestSubmitMode.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/SinkRequestSubmitMode.java
new file mode 100644
index 0000000..71e516d
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/config/SinkRequestSubmitMode.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.config;
+
+/** Sink request submit mode. */
+public enum SinkRequestSubmitMode {
+ SINGLE("single"),
+ BATCH("batch");
+
+ private final String mode;
+
+ SinkRequestSubmitMode(String mode) {
+ this.mode = mode;
+ }
+
+ public String getMode() {
+ return mode;
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/BasicAuthHeaderValuePreprocessor.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/BasicAuthHeaderValuePreprocessor.java
similarity index 51%
rename from src/main/java/com/getindata/connectors/http/internal/BasicAuthHeaderValuePreprocessor.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/BasicAuthHeaderValuePreprocessor.java
index 8c18c1c..22c53c8 100644
--- a/src/main/java/com/getindata/connectors/http/internal/BasicAuthHeaderValuePreprocessor.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/BasicAuthHeaderValuePreprocessor.java
@@ -1,11 +1,28 @@
-package com.getindata.connectors.http.internal;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.preprocessor;
import java.util.Base64;
import java.util.Objects;
/**
- * Header processor for HTTP Basic Authentication mechanism.
- * Only "Basic" authentication is supported currently.
+ * Header processor for HTTP Basic Authentication mechanism. Only "Basic" authentication is
+ * supported currently.
*/
public class BasicAuthHeaderValuePreprocessor implements HeaderValuePreprocessor {
@@ -14,8 +31,8 @@ public class BasicAuthHeaderValuePreprocessor implements HeaderValuePreprocessor
private boolean useRawAuthHeader = false;
/**
- * Creates a new instance of BasicAuthHeaderValuePreprocessor that uses
- * the default processing of the Authorization header.
+ * Creates a new instance of BasicAuthHeaderValuePreprocessor that uses the default processing
+ * of the Authorization header.
*/
public BasicAuthHeaderValuePreprocessor() {
this(false);
@@ -25,8 +42,7 @@ public class BasicAuthHeaderValuePreprocessor implements HeaderValuePreprocessor
* Creates a new instance of BasicAuthHeaderValuePreprocessor.
*
* @param useRawAuthHeader If set to true, the Authorization header is kept as-is,
- * untransformed. Otherwise, uses the default processing of the
- * Authorization header.
+ * untransformed. Otherwise, uses the default processing of the Authorization header.
*/
public BasicAuthHeaderValuePreprocessor(boolean useRawAuthHeader) {
this.useRawAuthHeader = useRawAuthHeader;
@@ -35,14 +51,14 @@ public class BasicAuthHeaderValuePreprocessor implements HeaderValuePreprocessor
/**
* Calculates {@link Base64} value of provided header value. For Basic authentication mechanism,
* the raw value is expected to match user:password pattern.
- * <p>
- * If rawValue starts with "Basic " prefix, or useRawAuthHeader has been set to true, it is
+ *
+ * <p>If rawValue starts with "Basic " prefix, or useRawAuthHeader has been set to true, it is
* assumed that this value is already converted to the expected "Authorization" header value.
*
* @param rawValue header original value to modify.
* @return value of "Authorization" header with format "Basic " + Base64 from rawValue or
- * rawValue without any changes if it starts with "Basic " prefix or useRawAuthHeader is
- * set to true.
+ * rawValue without any changes if it starts with "Basic " prefix or useRawAuthHeader is set
+ * to true.
*/
@Override
public String preprocessHeaderValue(String rawValue) {
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/ComposeHeaderPreprocessor.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/ComposeHeaderPreprocessor.java
new file mode 100644
index 0000000..142e27f
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/ComposeHeaderPreprocessor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.preprocessor;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This implementation of {@link HeaderPreprocessor} acts as a registry for all {@link
+ * HeaderValuePreprocessor} that should be applied on HTTP request.
+ */
+public class ComposeHeaderPreprocessor implements HeaderPreprocessor {
+
+ /**
+ * Default, pass through header value preprocessor used whenever dedicated preprocessor for a
+ * given header does not exist.
+ */
+ private static final HeaderValuePreprocessor DEFAULT_VALUE_PREPROCESSOR = rawValue -> rawValue;
+
+ /** Map with {@link HeaderValuePreprocessor} to apply. */
+ private final Map<String, HeaderValuePreprocessor> valuePreprocessors;
+
+ /**
+ * Creates a new instance of ComposeHeaderPreprocessor for provided {@link
+ * HeaderValuePreprocessor} map.
+ *
+ * @param valuePreprocessors map of {@link HeaderValuePreprocessor} that should be used for this
+ * processor. If null, then default, pass through header value processor will be used for
+ * every header.
+ */
+ public ComposeHeaderPreprocessor(Map<String, HeaderValuePreprocessor> valuePreprocessors) {
+ this.valuePreprocessors =
+ (valuePreprocessors == null)
+ ? Collections.emptyMap()
+ : new HashMap<>(valuePreprocessors);
+ }
+
+ @Override
+ public String preprocessValueForHeader(String headerName, String headerRawValue) {
+ return valuePreprocessors
+ .getOrDefault(headerName, DEFAULT_VALUE_PREPROCESSOR)
+ .preprocessHeaderValue(headerRawValue);
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderPreprocessor.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderPreprocessor.java
new file mode 100644
index 0000000..2f5f82a
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderPreprocessor.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.preprocessor;
+
+import java.io.Serializable;
+
+/** Interface for header preprocessing. */
+public interface HeaderPreprocessor extends Serializable {
+
+ /**
+ * Preprocess value of a header.Preprocessing can change or validate header value.
+ *
+ * @param headerName header name which value should be preprocessed.
+ * @param headerRawValue header value to process.
+ * @return preprocessed header value.
+ */
+ String preprocessValueForHeader(String headerName, String headerRawValue);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderValuePreprocessor.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderValuePreprocessor.java
new file mode 100644
index 0000000..94d3c80
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/HeaderValuePreprocessor.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.preprocessor;
+
+import java.io.Serializable;
+
+/**
+ * Processor interface which modifies header value based on implemented logic. An example would be
+ * calculation of Value of Authorization header.
+ */
+public interface HeaderValuePreprocessor extends Serializable {
+
+ /**
+ * Modifies header rawValue according to the implemented logic.
+ *
+ * @param rawValue header original value to modify
+ * @return modified header value.
+ */
+ String preprocessHeaderValue(String rawValue);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/OIDCAuthHeaderValuePreprocessor.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/OIDCAuthHeaderValuePreprocessor.java
new file mode 100644
index 0000000..a032f98
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/preprocessor/OIDCAuthHeaderValuePreprocessor.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.preprocessor;
+
+import org.apache.flink.connector.http.auth.OidcAccessTokenManager;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.http.HttpClient;
+import java.time.Duration;
+import java.util.Optional;
+
+/** Header processor for HTTP OIDC Authentication mechanism. */
+@Slf4j
+public class OIDCAuthHeaderValuePreprocessor implements HeaderValuePreprocessor {
+
+ private final String oidcAuthURL;
+ private final String oidcTokenRequest;
+ private Duration oidcExpiryReduction = Duration.ofSeconds(1);
+
+ /**
+ * Add the access token to the request using OidcAuth authenticate method that gives us a valid
+ * access token.
+ *
+ * @param oidcAuthURL OIDC token endpoint
+ * @param oidcTokenRequest OIDC Token Request
+ * @param oidcExpiryReduction OIDC token expiry reduction
+ */
+ public OIDCAuthHeaderValuePreprocessor(
+ String oidcAuthURL, String oidcTokenRequest, Optional<Duration> oidcExpiryReduction) {
+ this.oidcAuthURL = oidcAuthURL;
+ this.oidcTokenRequest = oidcTokenRequest;
+ if (oidcExpiryReduction.isPresent()) {
+ this.oidcExpiryReduction = oidcExpiryReduction.get();
+ }
+ }
+
+ @Override
+ public String preprocessHeaderValue(String rawValue) {
+ OidcAccessTokenManager auth =
+ new OidcAccessTokenManager(
+ HttpClient.newBuilder().build(),
+ oidcTokenRequest,
+ oidcAuthURL,
+ oidcExpiryReduction);
+ // apply the OIDC authentication by adding the dynamically calculated header value.
+ return "BEARER " + auth.authenticate();
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/HttpClientWithRetry.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/HttpClientWithRetry.java
new file mode 100644
index 0000000..e3c30c5
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/HttpClientWithRetry.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.retry;
+
+import org.apache.flink.connector.http.HttpStatusCodeValidationFailedException;
+import org.apache.flink.connector.http.status.HttpResponseChecker;
+import org.apache.flink.metrics.MetricGroup;
+
+import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.function.Supplier;
+
+/** {@link HttpClient} with retry. */
+@Slf4j
+public class HttpClientWithRetry {
+
+ private final HttpClient httpClient;
+ @Getter private final HttpResponseChecker responseChecker;
+ private final Retry retry;
+
+ @Builder
+ HttpClientWithRetry(
+ HttpClient httpClient, RetryConfig retryConfig, HttpResponseChecker responseChecker) {
+ this.httpClient = httpClient;
+ this.responseChecker = responseChecker;
+ var adjustedRetryConfig =
+ RetryConfig.from(retryConfig)
+ .retryExceptions(IOException.class)
+ .retryOnResult(this::isTemporalError)
+ .build();
+ this.retry = Retry.of("http-lookup-connector", adjustedRetryConfig);
+ }
+
+ public void registerMetrics(MetricGroup metrics) {
+ var group = metrics.addGroup("http_lookup_connector");
+ group.gauge(
+ "successfulCallsWithRetryAttempt",
+ () -> retry.getMetrics().getNumberOfSuccessfulCallsWithRetryAttempt());
+ group.gauge(
+ "successfulCallsWithoutRetryAttempt",
+ () -> retry.getMetrics().getNumberOfSuccessfulCallsWithoutRetryAttempt());
+ }
+
+ public <T> HttpResponse<T> send(
+ Supplier<HttpRequest> requestSupplier, HttpResponse.BodyHandler<T> responseBodyHandler)
+ throws IOException, InterruptedException, HttpStatusCodeValidationFailedException {
+ try {
+ var response =
+ Retry.decorateCheckedSupplier(
+ retry,
+ () ->
+ httpClient.send(
+ requestSupplier.get(), responseBodyHandler))
+ .apply();
+ if (!responseChecker.isSuccessful(response)) {
+ throw new HttpStatusCodeValidationFailedException(
+ "Incorrect response code: " + response.statusCode(), response);
+ }
+ return response;
+ } catch (IOException | InterruptedException | HttpStatusCodeValidationFailedException e) {
+ throw e; // re-throw without wrapping
+ } catch (Throwable t) {
+ throw new RuntimeException("Unexpected exception", t);
+ }
+ }
+
+ private boolean isTemporalError(Object response) {
+ return responseChecker.isTemporalError((HttpResponse<?>) response);
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/retry/RetryConfigProvider.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryConfigProvider.java
similarity index 50%
rename from src/main/java/com/getindata/connectors/http/internal/retry/RetryConfigProvider.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryConfigProvider.java
index 0a09d7d..424113b 100644
--- a/src/main/java/com/getindata/connectors/http/internal/retry/RetryConfigProvider.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryConfigProvider.java
@@ -1,19 +1,37 @@
-package com.getindata.connectors.http.internal.retry;
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.retry;
+
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.table.connector.source.lookup.LookupOptions;
import io.github.resilience4j.core.IntervalFunction;
import io.github.resilience4j.retry.RetryConfig;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.flink.table.connector.source.lookup.LookupOptions;
-import static io.github.resilience4j.core.IntervalFunction.ofExponentialBackoff;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_INITIAL_BACKOFF;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MAX_BACKOFF;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MULTIPLIER;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_STRATEGY;
+import static io.github.resilience4j.core.IntervalFunction.ofExponentialBackoff;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_INITIAL_BACKOFF;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MAX_BACKOFF;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MULTIPLIER;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_STRATEGY;
+/** Configuration for Retry. */
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public class RetryConfigProvider {
@@ -24,9 +42,7 @@ public class RetryConfigProvider {
}
private RetryConfig create() {
- return createBuilder()
- .maxAttempts(config.get(LookupOptions.MAX_RETRIES) + 1)
- .build();
+ return createBuilder().maxAttempts(config.get(LookupOptions.MAX_RETRIES) + 1).build();
}
private RetryConfig.Builder<?> createBuilder() {
@@ -45,7 +61,8 @@ public class RetryConfigProvider {
private RetryConfig.Builder<?> configureFixedDelay() {
return RetryConfig.custom()
- .intervalFunction(IntervalFunction.of(config.get(SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY)));
+ .intervalFunction(
+ IntervalFunction.of(config.get(SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY)));
}
private RetryConfig.Builder<?> configureExponentialDelay() {
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryStrategyType.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryStrategyType.java
new file mode 100644
index 0000000..d3d21d9
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/retry/RetryStrategyType.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.retry;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+/** Retry strategy type enum. */
+@Getter
+@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+public enum RetryStrategyType {
+ FIXED_DELAY("fixed-delay"),
+ EXPONENTIAL_DELAY("exponential-delay"),
+ ;
+
+ private final String code;
+
+ public static RetryStrategyType fromCode(String code) {
+ if (code == null) {
+ throw new NullPointerException("Code is null");
+ }
+ for (var strategy : RetryStrategyType.values()) {
+ if (strategy.getCode().equalsIgnoreCase(code)) {
+ return strategy;
+ }
+ }
+ throw new IllegalArgumentException("No enum constant for " + code);
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/security/SecurityContext.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SecurityContext.java
similarity index 68%
rename from src/main/java/com/getindata/connectors/http/internal/security/SecurityContext.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SecurityContext.java
index 1fed044..35bb73d 100644
--- a/src/main/java/com/getindata/connectors/http/internal/security/SecurityContext.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SecurityContext.java
@@ -1,4 +1,30 @@
-package com.getindata.connectors.http.internal.security;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.security;
+
+import org.apache.flink.connector.http.utils.ConfigUtils;
+
+import lombok.extern.slf4j.Slf4j;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -17,14 +43,6 @@ import java.security.spec.PKCS8EncodedKeySpec;
import java.util.Base64;
import java.util.Collection;
import java.util.UUID;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-
-import lombok.extern.slf4j.Slf4j;
-
-import com.getindata.connectors.http.internal.utils.ConfigUtils;
/**
* This class represents a security context for given Http connector instance. The Security context
@@ -48,16 +66,15 @@ public class SecurityContext {
private final KeyStore keystore;
- /**
- * Creates instance of {@link SecurityContext} and initialize {@link KeyStore} instance.
- */
+ /** Creates instance of {@link SecurityContext} and initialize {@link KeyStore} instance. */
private SecurityContext(KeyStore keystore, char[] storePassword) {
this.keystore = keystore;
this.storePassword = storePassword;
}
/**
- * Creates a {@link SecurityContext} with empty {@link KeyStore}
+ * Creates a {@link SecurityContext} with empty {@link KeyStore}.
+ *
* @return new instance of {@link SecurityContext}
*/
public static SecurityContext create() {
@@ -71,14 +88,13 @@ public class SecurityContext {
return new SecurityContext(keystore, storePasswordCharArr);
} catch (Exception e) {
throw new RuntimeException(
- "Unable to create KeyStore for Http Connector Security Context.",
- e
- );
+ "Unable to create KeyStore for Http Connector Security Context.", e);
}
}
/**
* Creates a {@link SecurityContext} with {@link KeyStore} loaded from provided path.
+ *
* @param keyStorePath Path to keystore.
* @param storePassword password for keystore.
* @return new instance of {@link SecurityContext}
@@ -89,13 +105,11 @@ public class SecurityContext {
File file = new File(keyStorePath);
InputStream stream = new FileInputStream(file);
KeyStore keystore = KeyStore.getInstance(JKS_STORE_TYPE);
- keystore.load(stream,storePassword);
+ keystore.load(stream, storePassword);
return new SecurityContext(keystore, storePassword);
} catch (Exception e) {
throw new RuntimeException(
- "Unable to create KeyStore for Http Connector Security Context.",
- e
- );
+ "Unable to create KeyStore for Http Connector Security Context.", e);
}
}
@@ -122,10 +136,11 @@ public class SecurityContext {
}
/**
- * Creates TrustManagers for given {@link KeyStore} managed by this instance of
- * {@link SSLContext}. It is important that all keys and certificates should be added
- * before calling this method. Any key/certificate added after calling this method
- * will not be visible by previously created TrustManager objects.
+ * Creates TrustManagers for given {@link KeyStore} managed by this instance of {@link
+ * SSLContext}. It is important that all keys and certificates should be added before calling
+ * this method. Any key/certificate added after calling this method will not be visible by
+ * previously created TrustManager objects.
+ *
* @return an array of {@link TrustManager}
*/
public TrustManager[] getTrustManagers() {
@@ -138,39 +153,38 @@ public class SecurityContext {
return trustManagerFactory.getTrustManagers();
} catch (GeneralSecurityException e) {
throw new RuntimeException(
- "Unable to created Trust Managers for Http Connector security context.",
- e
- );
+ "Unable to created Trust Managers for Http Connector security context.", e);
}
}
/**
* Adds certificate to as trusted. Certificate is added only to this Context's {@link KeyStore}
* and not for entire JVM.
+ *
* @param certPath path to certificate that should be added as trusted.
*/
public void addCertToTrustStore(String certPath) {
log.info("Trying to add certificate to Security Context - " + certPath);
try (FileInputStream certInputStream = new FileInputStream(certPath)) {
- CertificateFactory certificateFactory = CertificateFactory.getInstance(
- X_509_CERTIFICATE_TYPE);
+ CertificateFactory certificateFactory =
+ CertificateFactory.getInstance(X_509_CERTIFICATE_TYPE);
Certificate certificate = certificateFactory.generateCertificate(certInputStream);
this.keystore.setCertificateEntry(UUID.randomUUID().toString(), certificate);
log.info("Certificated added to keyStore ass trusted - " + certPath);
} catch (Exception e) {
throw new RuntimeException(
- "Unable to add certificate as trusted to Http Connector security context - "
- + certPath,
- e
- );
+ "Unable to add certificate as trusted to Http Connector security context - "
+ + certPath,
+ e);
}
}
/**
* Add certificate and private key that should be used by anny Http Connector instance that uses
- * this {@link SSLContext} instance. Certificate and key are added only to this Context's
- * {@link KeyStore} and not for entire JVM.
+ * this {@link SSLContext} instance. Certificate and key are added only to this Context's {@link
+ * KeyStore} and not for entire JVM.
+ *
* @param publicKeyPath path to public key/certificate used for mTLS.
* @param privateKeyPath path to private key used for mTLS.
*/
@@ -182,31 +196,30 @@ public class SecurityContext {
byte[] decodedPrivateData = decodePrivateData(privateKeyPath, privateData);
CertificateFactory certificateFactory =
- CertificateFactory.getInstance(X_509_CERTIFICATE_TYPE);
- Collection<? extends Certificate> chain = certificateFactory.generateCertificates(
- new ByteArrayInputStream(publicData));
+ CertificateFactory.getInstance(X_509_CERTIFICATE_TYPE);
+ Collection<? extends Certificate> chain =
+ certificateFactory.generateCertificates(new ByteArrayInputStream(publicData));
- Key key = KeyFactory
- .getInstance(KEY_ALGORITHM)
- .generatePrivate(new PKCS8EncodedKeySpec(decodedPrivateData));
+ Key key =
+ KeyFactory.getInstance(KEY_ALGORITHM)
+ .generatePrivate(new PKCS8EncodedKeySpec(decodedPrivateData));
this.keystore.setKeyEntry(
- UUID.randomUUID().toString(),
- key,
- this.storePassword,
- chain.toArray(new Certificate[0])
- );
+ UUID.randomUUID().toString(),
+ key,
+ this.storePassword,
+ chain.toArray(new Certificate[0]));
} catch (Exception e) {
throw new RuntimeException(
- "Unable to add client private key/public certificate to Http Connector KeyStore. "
- + String.join(",", privateKeyPath, publicKeyPath),
- e
- );
+ "Unable to add client private key/public certificate to Http Connector KeyStore. "
+ + String.join(",", privateKeyPath, publicKeyPath),
+ e);
}
}
/**
* Reads private key data. Key can be in PEM and DER coding and in PKCS8 format.
+ *
* @param privateKeyPath path to private key.
* @param privateData read bytes from private key,
* @return decoded key data.
@@ -217,10 +230,11 @@ public class SecurityContext {
// openssl pkcs8 -topk8 -inform PEM -outform PEM -in client.pem
// -out clientPrivateKey.pem -nocrypt
if (privateKeyPath.endsWith(".pem")) {
- String privateString = new String(privateData, Charset.defaultCharset())
- .replace(PRIVATE_KEY_HEADER, "")
- .replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "")
- .replace(PRIVATE_KEY_FOOTER, "");
+ String privateString =
+ new String(privateData, Charset.defaultCharset())
+ .replace(PRIVATE_KEY_HEADER, "")
+ .replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, "")
+ .replace(PRIVATE_KEY_FOOTER, "");
return Base64.getDecoder().decode(privateString);
} else {
diff --git a/src/main/java/com/getindata/connectors/http/internal/security/SelfSignedTrustManager.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SelfSignedTrustManager.java
similarity index 59%
rename from src/main/java/com/getindata/connectors/http/internal/security/SelfSignedTrustManager.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SelfSignedTrustManager.java
index 56c9a4b..51684eb 100644
--- a/src/main/java/com/getindata/connectors/http/internal/security/SelfSignedTrustManager.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/security/SelfSignedTrustManager.java
@@ -1,14 +1,33 @@
-package com.getindata.connectors.http.internal.security;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.security;
+
+import lombok.extern.slf4j.Slf4j;
-import java.net.Socket;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.X509ExtendedTrustManager;
import javax.net.ssl.X509TrustManager;
-import lombok.extern.slf4j.Slf4j;
+import java.net.Socket;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+/** Self signed trust manager. */
@Slf4j
public class SelfSignedTrustManager extends X509ExtendedTrustManager {
@@ -23,12 +42,12 @@ public class SelfSignedTrustManager extends X509ExtendedTrustManager {
}
public void checkClientTrusted(X509Certificate[] chain, String s, Socket socket)
- throws CertificateException {
+ throws CertificateException {
this.delegate.checkClientTrusted(chain, s);
}
public void checkClientTrusted(X509Certificate[] chain, String s, SSLEngine sslEngine)
- throws CertificateException {
+ throws CertificateException {
this.delegate.checkClientTrusted(chain, s);
}
@@ -37,12 +56,12 @@ public class SelfSignedTrustManager extends X509ExtendedTrustManager {
}
public void checkServerTrusted(X509Certificate[] chain, String s, Socket socket)
- throws CertificateException {
+ throws CertificateException {
log.info("Allowing self signed server certificates.");
}
public void checkServerTrusted(X509Certificate[] chain, String s, SSLEngine sslEngine)
- throws CertificateException {
+ throws CertificateException {
log.info("Allowing self signed server certificates.");
}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkInternal.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkInternal.java
new file mode 100644
index 0000000..a597cb1
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkInternal.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink;
+
+import org.apache.flink.connector.base.sink.AsyncSinkBase;
+import org.apache.flink.connector.base.sink.writer.BufferedRequestState;
+import org.apache.flink.connector.base.sink.writer.ElementConverter;
+import org.apache.flink.connector.http.HttpPostRequestCallback;
+import org.apache.flink.connector.http.HttpSink;
+import org.apache.flink.connector.http.HttpSinkBuilder;
+import org.apache.flink.connector.http.SchemaLifecycleAwareElementConverter;
+import org.apache.flink.connector.http.clients.SinkHttpClientBuilder;
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.config.SinkRequestSubmitMode;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.sink.httpclient.BatchRequestSubmitterFactory;
+import org.apache.flink.connector.http.sink.httpclient.HttpRequest;
+import org.apache.flink.connector.http.sink.httpclient.PerRequestRequestSubmitterFactory;
+import org.apache.flink.connector.http.sink.httpclient.RequestSubmitterFactory;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Properties;
+
+/**
+ * An internal implementation of HTTP Sink that performs async requests against a specified HTTP
+ * endpoint using the buffering protocol specified in {@link AsyncSinkBase}.
+ *
+ * <p>API of this class can change without any concerns as long as it does not have any influence on
+ * methods defined in {@link HttpSink} and {@link HttpSinkBuilder} classes.
+ *
+ * <p>The behaviour of the buffering may be specified by providing configuration during the sink
+ * build time.
+ *
+ * <ul>
+ * <li>{@code maxBatchSize}: the maximum size of a batch of entries that may be sent to the HTTP
+ * endpoint;
+ * <li>{@code maxInFlightRequests}: the maximum number of in flight requests that may exist, if
+ * any more in flight requests need to be initiated once the maximum has been reached, then it
+ * will be blocked until some have completed;
+ * <li>{@code maxBufferedRequests}: the maximum number of elements held in the buffer, requests to
+ * add elements will be blocked while the number of elements in the buffer is at the maximum;
+ * <li>{@code maxBatchSizeInBytes}: the maximum size of a batch of entries that may be sent to the
+ * HTTP endpoint measured in bytes;
+ * <li>{@code maxTimeInBufferMS}: the maximum amount of time an entry is allowed to live in the
+ * buffer, if any element reaches this age, the entire buffer will be flushed immediately;
+ * <li>{@code maxRecordSizeInBytes}: the maximum size of a record the sink will accept into the
+ * buffer, a record of size larger than this will be rejected when passed to the sink.
+ * <li>{@code httpPostRequestCallback}: the {@link HttpPostRequestCallback} implementation for
+ * processing of requests and responses;
+ * <li>{@code properties}: properties related to the Http Sink.
+ * </ul>
+ *
+ * @param <InputT> type of the elements that should be sent through HTTP request.
+ */
+public class HttpSinkInternal<InputT> extends AsyncSinkBase<InputT, HttpSinkRequestEntry> {
+
+ private final String endpointUrl;
+
+ // having Builder instead of an instance of `SinkHttpClient`
+ // makes it possible to serialize `HttpSink`
+ private final SinkHttpClientBuilder sinkHttpClientBuilder;
+
+ private final HttpPostRequestCallback<HttpRequest> httpPostRequestCallback;
+
+ private final HeaderPreprocessor headerPreprocessor;
+
+ private final Properties properties;
+
+ protected HttpSinkInternal(
+ ElementConverter<InputT, HttpSinkRequestEntry> elementConverter,
+ int maxBatchSize,
+ int maxInFlightRequests,
+ int maxBufferedRequests,
+ long maxBatchSizeInBytes,
+ long maxTimeInBufferMS,
+ long maxRecordSizeInBytes,
+ String endpointUrl,
+ HttpPostRequestCallback<HttpRequest> httpPostRequestCallback,
+ HeaderPreprocessor headerPreprocessor,
+ SinkHttpClientBuilder sinkHttpClientBuilder,
+ Properties properties) {
+
+ super(
+ elementConverter,
+ maxBatchSize,
+ maxInFlightRequests,
+ maxBufferedRequests,
+ maxBatchSizeInBytes,
+ maxTimeInBufferMS,
+ maxRecordSizeInBytes);
+
+ Preconditions.checkArgument(
+ !StringUtils.isNullOrWhitespaceOnly(endpointUrl),
+ "The endpoint URL must be set when initializing HTTP Sink.");
+ this.endpointUrl = endpointUrl;
+ this.httpPostRequestCallback =
+ Preconditions.checkNotNull(
+ httpPostRequestCallback,
+ "Post request callback must be set when initializing HTTP Sink.");
+ this.headerPreprocessor =
+ Preconditions.checkNotNull(
+ headerPreprocessor,
+ "Header Preprocessor must be set when initializing HTTP Sink.");
+ this.sinkHttpClientBuilder =
+ Preconditions.checkNotNull(
+ sinkHttpClientBuilder,
+ "The HTTP client builder must not be null when initializing HTTP Sink.");
+ this.properties = properties;
+ }
+
+ @Override
+ public StatefulSinkWriter<InputT, BufferedRequestState<HttpSinkRequestEntry>> createWriter(
+ InitContext context) throws IOException {
+
+ ElementConverter<InputT, HttpSinkRequestEntry> elementConverter = getElementConverter();
+ if (elementConverter instanceof SchemaLifecycleAwareElementConverter) {
+ // This cast is needed for Flink 1.15.3 build
+ ((SchemaLifecycleAwareElementConverter<?, ?>) elementConverter).open(context);
+ }
+
+ return new HttpSinkWriter<>(
+ elementConverter,
+ context,
+ getMaxBatchSize(),
+ getMaxInFlightRequests(),
+ getMaxBufferedRequests(),
+ getMaxBatchSizeInBytes(),
+ getMaxTimeInBufferMS(),
+ getMaxRecordSizeInBytes(),
+ endpointUrl,
+ sinkHttpClientBuilder.build(
+ properties,
+ httpPostRequestCallback,
+ headerPreprocessor,
+ getRequestSubmitterFactory()),
+ Collections.emptyList(),
+ properties);
+ }
+
+ @Override
+ public StatefulSinkWriter<InputT, BufferedRequestState<HttpSinkRequestEntry>> restoreWriter(
+ InitContext context,
+ Collection<BufferedRequestState<HttpSinkRequestEntry>> recoveredState)
+ throws IOException {
+
+ return new HttpSinkWriter<>(
+ getElementConverter(),
+ context,
+ getMaxBatchSize(),
+ getMaxInFlightRequests(),
+ getMaxBufferedRequests(),
+ getMaxBatchSizeInBytes(),
+ getMaxTimeInBufferMS(),
+ getMaxRecordSizeInBytes(),
+ endpointUrl,
+ sinkHttpClientBuilder.build(
+ properties,
+ httpPostRequestCallback,
+ headerPreprocessor,
+ getRequestSubmitterFactory()),
+ recoveredState,
+ properties);
+ }
+
+ @Override
+ public SimpleVersionedSerializer<BufferedRequestState<HttpSinkRequestEntry>>
+ getWriterStateSerializer() {
+ return new HttpSinkWriterStateSerializer();
+ }
+
+ private RequestSubmitterFactory getRequestSubmitterFactory() {
+
+ if (SinkRequestSubmitMode.SINGLE
+ .getMode()
+ .equalsIgnoreCase(
+ properties.getProperty(
+ HttpConnectorConfigConstants.SINK_HTTP_REQUEST_MODE))) {
+ return new PerRequestRequestSubmitterFactory();
+ }
+ return new BatchRequestSubmitterFactory(getMaxBatchSize());
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkRequestEntry.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkRequestEntry.java
new file mode 100644
index 0000000..0b61e17
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkRequestEntry.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink;
+
+import org.apache.flink.connector.http.HttpSink;
+
+import lombok.EqualsAndHashCode;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+
+import java.io.Serializable;
+
+/**
+ * Represents a single {@link HttpSink} request. Contains the HTTP method name, Content-Type header
+ * value, and byte representation of the body of the request.
+ */
+@RequiredArgsConstructor
+@EqualsAndHashCode
+@ToString
+public final class HttpSinkRequestEntry implements Serializable {
+
+ /** HTTP method name to use when sending the request. */
+ @NonNull public final String method;
+
+ /** Body of the request, encoded as byte array. */
+ public final byte[] element;
+
+ /** @return the size of the {@link HttpSinkRequestEntry#element} */
+ public long getSizeInBytes() {
+ return element.length;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriter.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriter.java
new file mode 100644
index 0000000..5dbc6cd
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriter.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink;
+
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.sink.writer.AsyncSinkWriter;
+import org.apache.flink.connector.base.sink.writer.BufferedRequestState;
+import org.apache.flink.connector.base.sink.writer.ElementConverter;
+import org.apache.flink.connector.http.HttpSink;
+import org.apache.flink.connector.http.clients.SinkHttpClient;
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.utils.ThreadUtils;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.function.Consumer;
+
+/**
+ * Sink writer created by {@link HttpSink} to write to an HTTP endpoint.
+ *
+ * <p>More details on the internals of this sink writer may be found in {@link AsyncSinkWriter}
+ * documentation.
+ *
+ * @param <InputT> type of the elements that should be sent through HTTP request.
+ */
+@Slf4j
+public class HttpSinkWriter<InputT> extends AsyncSinkWriter<InputT, HttpSinkRequestEntry> {
+
+ private static final String HTTP_SINK_WRITER_THREAD_POOL_SIZE = "4";
+
+ /** Thread pool to handle HTTP response from HTTP client. */
+ private final ExecutorService sinkWriterThreadPool;
+
+ private final String endpointUrl;
+
+ private final SinkHttpClient sinkHttpClient;
+
+ private final Counter numRecordsSendErrorsCounter;
+
+ public HttpSinkWriter(
+ ElementConverter<InputT, HttpSinkRequestEntry> elementConverter,
+ Sink.InitContext context,
+ int maxBatchSize,
+ int maxInFlightRequests,
+ int maxBufferedRequests,
+ long maxBatchSizeInBytes,
+ long maxTimeInBufferMS,
+ long maxRecordSizeInBytes,
+ String endpointUrl,
+ SinkHttpClient sinkHttpClient,
+ Collection<BufferedRequestState<HttpSinkRequestEntry>> bufferedRequestStates,
+ Properties properties) {
+
+ super(
+ elementConverter,
+ context,
+ maxBatchSize,
+ maxInFlightRequests,
+ maxBufferedRequests,
+ maxBatchSizeInBytes,
+ maxTimeInBufferMS,
+ maxRecordSizeInBytes,
+ bufferedRequestStates);
+ this.endpointUrl = endpointUrl;
+ this.sinkHttpClient = sinkHttpClient;
+
+ var metrics = context.metricGroup();
+ this.numRecordsSendErrorsCounter = metrics.getNumRecordsSendErrorsCounter();
+
+ int sinkWriterThreadPollSize =
+ Integer.parseInt(
+ properties.getProperty(
+ HttpConnectorConfigConstants.SINK_HTTP_WRITER_THREAD_POOL_SIZE,
+ HTTP_SINK_WRITER_THREAD_POOL_SIZE));
+
+ this.sinkWriterThreadPool =
+ Executors.newFixedThreadPool(
+ sinkWriterThreadPollSize,
+ new ExecutorThreadFactory(
+ "http-sink-writer-worker", ThreadUtils.LOGGING_EXCEPTION_HANDLER));
+ }
+
+ // TODO: Reintroduce retries by adding backoff policy
+ @Override
+ protected void submitRequestEntries(
+ List<HttpSinkRequestEntry> requestEntries,
+ Consumer<List<HttpSinkRequestEntry>> requestResult) {
+ var future = sinkHttpClient.putRequests(requestEntries, endpointUrl);
+ future.whenCompleteAsync(
+ (response, err) -> {
+ if (err != null) {
+ int failedRequestsNumber = requestEntries.size();
+ log.error(
+ "Http Sink fatally failed to write all {} requests",
+ failedRequestsNumber);
+ numRecordsSendErrorsCounter.inc(failedRequestsNumber);
+
+ // TODO: Make `HttpSinkInternal` retry the failed requests.
+ // Currently, it does not retry those at all, only adds their count
+ // to the `numRecordsSendErrors` metric. It is due to the fact we do not
+ // have
+ // a clear image how we want to do it, so it would be both efficient and
+ // correct.
+ // requestResult.accept(requestEntries);
+ } else if (response.getFailedRequests().size() > 0) {
+ int failedRequestsNumber = response.getFailedRequests().size();
+ log.error(
+ "Http Sink failed to write and will retry {} requests",
+ failedRequestsNumber);
+ numRecordsSendErrorsCounter.inc(failedRequestsNumber);
+
+ // TODO: Make `HttpSinkInternal` retry the failed requests. Currently,
+ // it does not retry those at all, only adds their count to the
+ // `numRecordsSendErrors` metric. It is due to the fact we do not have
+ // a clear image how we want to do it, so it would be both efficient and
+ // correct.
+
+ // requestResult.accept(response.getFailedRequests());
+ // } else {
+ // requestResult.accept(Collections.emptyList());
+ // }
+ }
+ requestResult.accept(Collections.emptyList());
+ },
+ sinkWriterThreadPool);
+ }
+
+ @Override
+ protected long getSizeInBytes(HttpSinkRequestEntry s) {
+ return s.getSizeInBytes();
+ }
+
+ @Override
+ public void close() {
+ sinkWriterThreadPool.shutdownNow();
+ super.close();
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriterStateSerializer.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriterStateSerializer.java
new file mode 100644
index 0000000..c41e5e9
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/HttpSinkWriterStateSerializer.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink;
+
+import org.apache.flink.connector.base.sink.writer.AsyncSinkWriterStateSerializer;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+/**
+ * An implementation of {@link AsyncSinkWriterStateSerializer} for {@link HttpSinkInternal} and its
+ * {@link HttpSinkWriter}.
+ */
+public class HttpSinkWriterStateSerializer
+ extends AsyncSinkWriterStateSerializer<HttpSinkRequestEntry> {
+
+ @Override
+ protected void serializeRequestToStream(HttpSinkRequestEntry s, DataOutputStream out)
+ throws IOException {
+ out.writeUTF(s.method);
+ out.write(s.element);
+ }
+
+ @Override
+ protected HttpSinkRequestEntry deserializeRequestFromStream(
+ long requestSize, DataInputStream in) throws IOException {
+ var method = in.readUTF();
+ var bytes = new byte[(int) requestSize];
+ in.read(bytes);
+ return new HttpSinkRequestEntry(method, bytes);
+ }
+
+ @Override
+ public int getVersion() {
+ return 1;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/AbstractRequestSubmitter.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/AbstractRequestSubmitter.java
new file mode 100644
index 0000000..10f9906
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/AbstractRequestSubmitter.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.utils.ThreadUtils;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import java.net.http.HttpClient;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/** Abstract request submitter. */
+public abstract class AbstractRequestSubmitter implements RequestSubmitter {
+
+ protected static final int HTTP_CLIENT_PUBLISHING_THREAD_POOL_SIZE = 1;
+
+ protected static final String DEFAULT_REQUEST_TIMEOUT_SECONDS = "30";
+
+ /** Thread pool to handle HTTP response from HTTP client. */
+ protected final ExecutorService publishingThreadPool;
+
+ protected final int httpRequestTimeOutSeconds;
+
+ protected final String[] headersAndValues;
+
+ protected final HttpClient httpClient;
+
+ public AbstractRequestSubmitter(
+ Properties properties, String[] headersAndValues, HttpClient httpClient) {
+
+ this.headersAndValues = headersAndValues;
+ this.publishingThreadPool =
+ Executors.newFixedThreadPool(
+ HTTP_CLIENT_PUBLISHING_THREAD_POOL_SIZE,
+ new ExecutorThreadFactory(
+ "http-sink-client-response-worker",
+ ThreadUtils.LOGGING_EXCEPTION_HANDLER));
+
+ this.httpRequestTimeOutSeconds =
+ Integer.parseInt(
+ properties.getProperty(
+ HttpConnectorConfigConstants.SINK_HTTP_TIMEOUT_SECONDS,
+ DEFAULT_REQUEST_TIMEOUT_SECONDS));
+
+ this.httpClient = httpClient;
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/sink/httpclient/BatchRequestSubmitter.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitter.java
similarity index 60%
rename from src/main/java/com/getindata/connectors/http/internal/sink/httpclient/BatchRequestSubmitter.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitter.java
index 5165233..19b45d6 100644
--- a/src/main/java/com/getindata/connectors/http/internal/sink/httpclient/BatchRequestSubmitter.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitter.java
@@ -1,4 +1,27 @@
-package com.getindata.connectors.http.internal.sink.httpclient;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+
+import lombok.extern.slf4j.Slf4j;
import java.net.URI;
import java.net.http.HttpClient;
@@ -15,12 +38,6 @@ import java.util.List;
import java.util.Properties;
import java.util.concurrent.CompletableFuture;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.flink.annotation.VisibleForTesting;
-
-import com.getindata.connectors.http.internal.config.HttpConnectorConfigConstants;
-import com.getindata.connectors.http.internal.sink.HttpSinkRequestEntry;
-
/**
* This implementation groups received events in batches and submits each batch as individual HTTP
* requests. Batch is created based on batch size or based on HTTP method type.
@@ -37,21 +54,19 @@ public class BatchRequestSubmitter extends AbstractRequestSubmitter {
private final int httpRequestBatchSize;
public BatchRequestSubmitter(
- Properties properties,
- String[] headersAndValue,
- HttpClient httpClient) {
+ Properties properties, String[] headersAndValue, HttpClient httpClient) {
super(properties, headersAndValue, httpClient);
- this.httpRequestBatchSize = Integer.parseInt(
- properties.getProperty(HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE)
- );
+ this.httpRequestBatchSize =
+ Integer.parseInt(
+ properties.getProperty(
+ HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE));
}
@Override
public List<CompletableFuture<JavaNetHttpResponseWrapper>> submit(
- String endpointUrl,
- List<HttpSinkRequestEntry> requestsToSubmit) {
+ String endpointUrl, List<HttpSinkRequestEntry> requestsToSubmit) {
if (requestsToSubmit.isEmpty()) {
return Collections.emptyList();
@@ -63,7 +78,7 @@ public class BatchRequestSubmitter extends AbstractRequestSubmitter {
for (var entry : requestsToSubmit) {
if (requestBatch.size() == httpRequestBatchSize
- || !previousReqeustMethod.equalsIgnoreCase(entry.method)) {
+ || !previousReqeustMethod.equalsIgnoreCase(entry.method)) {
// break batch and submit
responseFutures.add(sendBatch(endpointUrl, requestBatch));
requestBatch.clear();
@@ -83,23 +98,21 @@ public class BatchRequestSubmitter extends AbstractRequestSubmitter {
}
private CompletableFuture<JavaNetHttpResponseWrapper> sendBatch(
- String endpointUrl,
- List<HttpSinkRequestEntry> reqeustBatch) {
+ String endpointUrl, List<HttpSinkRequestEntry> reqeustBatch) {
HttpRequest httpRequest = buildHttpRequest(reqeustBatch, URI.create(endpointUrl));
return httpClient
- .sendAsync(
- httpRequest.getHttpRequest(),
- HttpResponse.BodyHandlers.ofString())
- .exceptionally(ex -> {
- // TODO This will be executed on a ForkJoinPool Thread... refactor this someday.
- log.error("Request fatally failed because of an exception", ex);
- return null;
- })
- .thenApplyAsync(
- res -> new JavaNetHttpResponseWrapper(httpRequest, res),
- publishingThreadPool
- );
+ .sendAsync(httpRequest.getHttpRequest(), HttpResponse.BodyHandlers.ofString())
+ .exceptionally(
+ ex -> {
+ // TODO This will be executed on a ForkJoinPool Thread... refactor this
+ // someday.
+ log.error("Request fatally failed because of an exception", ex);
+ return null;
+ })
+ .thenApplyAsync(
+ res -> new JavaNetHttpResponseWrapper(httpRequest, res),
+ publishingThreadPool);
}
private HttpRequest buildHttpRequest(List<HttpSinkRequestEntry> reqeustBatch, URI endpointUri) {
@@ -121,12 +134,12 @@ public class BatchRequestSubmitter extends AbstractRequestSubmitter {
elements.set(elements.size() - 1, BATCH_END_BYTES);
publisher = BodyPublishers.ofByteArrays(elements);
- Builder requestBuilder = java.net.http.HttpRequest
- .newBuilder()
- .uri(endpointUri)
- .version(Version.HTTP_1_1)
- .timeout(Duration.ofSeconds(httpRequestTimeOutSeconds))
- .method(method, publisher);
+ Builder requestBuilder =
+ java.net.http.HttpRequest.newBuilder()
+ .uri(endpointUri)
+ .version(Version.HTTP_1_1)
+ .timeout(Duration.ofSeconds(httpRequestTimeOutSeconds))
+ .method(method, publisher);
if (headersAndValues.length != 0) {
requestBuilder.headers(headersAndValues);
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitterFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitterFactory.java
new file mode 100644
index 0000000..6e87fa1
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/BatchRequestSubmitterFactory.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.connector.http.config.ConfigException;
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.utils.JavaNetHttpClientFactory;
+import org.apache.flink.connector.http.utils.ThreadUtils;
+import org.apache.flink.util.StringUtils;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/** Batch request submitter factory. */
+public class BatchRequestSubmitterFactory implements RequestSubmitterFactory {
+
+ // TODO Add this property to config. Make sure to add note in README.md that will describe that
+ // any value greater than one will break order of messages.
+ static final int HTTP_CLIENT_THREAD_POOL_SIZE = 1;
+
+ private final String maxBatchSize;
+
+ public BatchRequestSubmitterFactory(int maxBatchSize) {
+ if (maxBatchSize < 1) {
+ throw new IllegalArgumentException(
+ "Batch Request submitter batch size must be greater than zero.");
+ }
+ this.maxBatchSize = String.valueOf(maxBatchSize);
+ }
+
+ @Override
+ public BatchRequestSubmitter createSubmitter(Properties properties, String[] headersAndValues) {
+ String batchRequestSize =
+ properties.getProperty(HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE);
+ if (StringUtils.isNullOrWhitespaceOnly(batchRequestSize)) {
+ properties.setProperty(
+ HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE, maxBatchSize);
+ } else {
+ try {
+ // TODO Create property validator someday.
+ int batchSize = Integer.parseInt(batchRequestSize);
+ if (batchSize < 1) {
+ throw new ConfigException(
+ String.format(
+ "Property %s must be greater than 0 but was: %s",
+ HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE,
+ batchRequestSize));
+ }
+ } catch (NumberFormatException e) {
+ // TODO Create property validator someday.
+ throw new ConfigException(
+ String.format(
+ "Property %s must be an integer but was: %s",
+ HttpConnectorConfigConstants.SINK_HTTP_BATCH_REQUEST_SIZE,
+ batchRequestSize),
+ e);
+ }
+ }
+
+ ExecutorService httpClientExecutor =
+ Executors.newFixedThreadPool(
+ HTTP_CLIENT_THREAD_POOL_SIZE,
+ new ExecutorThreadFactory(
+ "http-sink-client-batch-request-worker",
+ ThreadUtils.LOGGING_EXCEPTION_HANDLER));
+
+ return new BatchRequestSubmitter(
+ properties,
+ headersAndValues,
+ JavaNetHttpClientFactory.createClient(properties, httpClientExecutor));
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/HttpRequest.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/HttpRequest.java
new file mode 100644
index 0000000..c494e8f
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/HttpRequest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import lombok.Data;
+
+import java.util.List;
+
+/** Http request. */
+@Data
+public class HttpRequest {
+
+ public final java.net.http.HttpRequest httpRequest;
+
+ public final List<byte[]> elements;
+
+ public final String method;
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetHttpResponseWrapper.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetHttpResponseWrapper.java
new file mode 100644
index 0000000..a7b8b5f
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetHttpResponseWrapper.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.connector.http.HttpSink;
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+
+import lombok.Data;
+import lombok.NonNull;
+
+import java.net.http.HttpResponse;
+import java.util.Optional;
+
+/**
+ * A wrapper structure around an HTTP response, keeping a reference to a particular {@link
+ * HttpSinkRequestEntry}. Used internally by the {@code HttpSinkWriter} to pass {@code
+ * HttpSinkRequestEntry} along some other element that it is logically connected with.
+ */
+@Data
+final class JavaNetHttpResponseWrapper {
+
+ /** A representation of a single {@link HttpSink} request. */
+ @NonNull private final HttpRequest httpRequest;
+
+ /** A response to an HTTP request based on {@link HttpSinkRequestEntry}. */
+ private final HttpResponse<String> response;
+
+ public Optional<HttpResponse<String>> getResponse() {
+ return Optional.ofNullable(response);
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetSinkHttpClient.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetSinkHttpClient.java
new file mode 100644
index 0000000..6216288
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/JavaNetSinkHttpClient.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.connector.http.HttpPostRequestCallback;
+import org.apache.flink.connector.http.clients.SinkHttpClient;
+import org.apache.flink.connector.http.clients.SinkHttpClientResponse;
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+import org.apache.flink.connector.http.status.ComposeHttpStatusCodeChecker;
+import org.apache.flink.connector.http.status.ComposeHttpStatusCodeChecker.ComposeHttpStatusCodeCheckerConfig;
+import org.apache.flink.connector.http.status.HttpStatusCodeChecker;
+import org.apache.flink.connector.http.utils.HttpHeaderUtils;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.http.HttpClient;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * An implementation of {@link SinkHttpClient} that uses Java 11's {@link HttpClient}. This
+ * implementation supports HTTP traffic only.
+ */
+@Slf4j
+public class JavaNetSinkHttpClient implements SinkHttpClient {
+
+ private final String[] headersAndValues;
+
+ private final Map<String, String> headerMap;
+
+ private final HttpStatusCodeChecker statusCodeChecker;
+
+ private final HttpPostRequestCallback<HttpRequest> httpPostRequestCallback;
+
+ private final RequestSubmitter requestSubmitter;
+
+ public JavaNetSinkHttpClient(
+ Properties properties,
+ HttpPostRequestCallback<HttpRequest> httpPostRequestCallback,
+ HeaderPreprocessor headerPreprocessor,
+ RequestSubmitterFactory requestSubmitterFactory) {
+
+ this.httpPostRequestCallback = httpPostRequestCallback;
+ this.headerMap =
+ HttpHeaderUtils.prepareHeaderMap(
+ HttpConnectorConfigConstants.SINK_HEADER_PREFIX,
+ properties,
+ headerPreprocessor);
+
+ // TODO Inject this via constructor when implementing a response processor.
+ // Processor will be injected and it will wrap statusChecker implementation.
+ ComposeHttpStatusCodeCheckerConfig checkerConfig =
+ ComposeHttpStatusCodeCheckerConfig.builder()
+ .properties(properties)
+ .includeListPrefix(
+ HttpConnectorConfigConstants.HTTP_ERROR_SINK_CODE_INCLUDE_LIST)
+ .errorCodePrefix(HttpConnectorConfigConstants.HTTP_ERROR_SINK_CODES_LIST)
+ .build();
+
+ this.statusCodeChecker = new ComposeHttpStatusCodeChecker(checkerConfig);
+
+ this.headersAndValues = HttpHeaderUtils.toHeaderAndValueArray(this.headerMap);
+ this.requestSubmitter =
+ requestSubmitterFactory.createSubmitter(properties, headersAndValues);
+ }
+
+ @Override
+ public CompletableFuture<SinkHttpClientResponse> putRequests(
+ List<HttpSinkRequestEntry> requestEntries, String endpointUrl) {
+ return submitRequests(requestEntries, endpointUrl)
+ .thenApply(responses -> prepareSinkHttpClientResponse(responses, endpointUrl));
+ }
+
+ private CompletableFuture<List<JavaNetHttpResponseWrapper>> submitRequests(
+ List<HttpSinkRequestEntry> requestEntries, String endpointUrl) {
+
+ var responseFutures = requestSubmitter.submit(endpointUrl, requestEntries);
+ var allFutures = CompletableFuture.allOf(responseFutures.toArray(new CompletableFuture[0]));
+ return allFutures.thenApply(
+ _void ->
+ responseFutures.stream()
+ .map(CompletableFuture::join)
+ .collect(Collectors.toList()));
+ }
+
+ private SinkHttpClientResponse prepareSinkHttpClientResponse(
+ List<JavaNetHttpResponseWrapper> responses, String endpointUrl) {
+ var successfulResponses = new ArrayList<HttpRequest>();
+ var failedResponses = new ArrayList<HttpRequest>();
+
+ for (var response : responses) {
+ var sinkRequestEntry = response.getHttpRequest();
+ var optResponse = response.getResponse();
+
+ httpPostRequestCallback.call(
+ optResponse.orElse(null), sinkRequestEntry, endpointUrl, headerMap);
+
+ // TODO Add response processor here and orchestrate it with statusCodeChecker.
+ if (optResponse.isEmpty()
+ || statusCodeChecker.isErrorCode(optResponse.get().statusCode())) {
+ failedResponses.add(sinkRequestEntry);
+ } else {
+ successfulResponses.add(sinkRequestEntry);
+ }
+ }
+
+ return new SinkHttpClientResponse(successfulResponses, failedResponses);
+ }
+
+ @VisibleForTesting
+ String[] getHeadersAndValues() {
+ return Arrays.copyOf(headersAndValues, headersAndValues.length);
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestRequestSubmitterFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestRequestSubmitterFactory.java
new file mode 100644
index 0000000..f63361b
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestRequestSubmitterFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.connector.http.utils.JavaNetHttpClientFactory;
+import org.apache.flink.connector.http.utils.ThreadUtils;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/** Per request submitter factory. */
+public class PerRequestRequestSubmitterFactory implements RequestSubmitterFactory {
+
+ // TODO Add this property to config. Make sure to add note in README.md that will describe that
+ // any value greater than one will break order of messages.
+ static final int HTTP_CLIENT_THREAD_POOL_SIZE = 1;
+
+ @Override
+ public RequestSubmitter createSubmitter(Properties properties, String[] headersAndValues) {
+
+ ExecutorService httpClientExecutor =
+ Executors.newFixedThreadPool(
+ HTTP_CLIENT_THREAD_POOL_SIZE,
+ new ExecutorThreadFactory(
+ "http-sink-client-per-request-worker",
+ ThreadUtils.LOGGING_EXCEPTION_HANDLER));
+
+ return new PerRequestSubmitter(
+ properties,
+ headersAndValues,
+ JavaNetHttpClientFactory.createClient(properties, httpClientExecutor));
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestSubmitter.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestSubmitter.java
new file mode 100644
index 0000000..aee85c9
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/PerRequestSubmitter.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpClient.Version;
+import java.net.http.HttpRequest.BodyPublishers;
+import java.net.http.HttpRequest.Builder;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.CompletableFuture;
+
+/** This implementation creates HTTP requests for every processed event. */
+@Slf4j
+public class PerRequestSubmitter extends AbstractRequestSubmitter {
+
+ public PerRequestSubmitter(
+ Properties properties, String[] headersAndValues, HttpClient httpClient) {
+
+ super(properties, headersAndValues, httpClient);
+ }
+
+ @Override
+ public List<CompletableFuture<JavaNetHttpResponseWrapper>> submit(
+ String endpointUrl, List<HttpSinkRequestEntry> requestToSubmit) {
+
+ var endpointUri = URI.create(endpointUrl);
+ var responseFutures = new ArrayList<CompletableFuture<JavaNetHttpResponseWrapper>>();
+
+ for (var entry : requestToSubmit) {
+ HttpRequest httpRequest = buildHttpRequest(entry, endpointUri);
+ var response =
+ httpClient
+ .sendAsync(
+ httpRequest.getHttpRequest(),
+ HttpResponse.BodyHandlers.ofString())
+ .exceptionally(
+ ex -> {
+ // TODO This will be executed on a ForkJoinPool Thread...
+ // refactor this someday.
+ log.error(
+ "Request fatally failed because of an exception",
+ ex);
+ return null;
+ })
+ .thenApplyAsync(
+ res -> new JavaNetHttpResponseWrapper(httpRequest, res),
+ publishingThreadPool);
+ responseFutures.add(response);
+ }
+ return responseFutures;
+ }
+
+ private HttpRequest buildHttpRequest(HttpSinkRequestEntry requestEntry, URI endpointUri) {
+ Builder requestBuilder =
+ java.net.http.HttpRequest.newBuilder()
+ .uri(endpointUri)
+ .version(Version.HTTP_1_1)
+ .timeout(Duration.ofSeconds(httpRequestTimeOutSeconds))
+ .method(
+ requestEntry.method,
+ BodyPublishers.ofByteArray(requestEntry.element));
+
+ if (headersAndValues.length != 0) {
+ requestBuilder.headers(headersAndValues);
+ }
+
+ return new HttpRequest(
+ requestBuilder.build(), List.of(requestEntry.element), requestEntry.method);
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitter.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitter.java
new file mode 100644
index 0000000..9835248
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitter.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+/** Submits request via HTTP. */
+public interface RequestSubmitter {
+
+ List<CompletableFuture<JavaNetHttpResponseWrapper>> submit(
+ String endpointUrl, List<HttpSinkRequestEntry> requestToSubmit);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitterFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitterFactory.java
new file mode 100644
index 0000000..188298a
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/sink/httpclient/RequestSubmitterFactory.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.sink.httpclient;
+
+import java.util.Properties;
+
+/** Request submitter factory. */
+public interface RequestSubmitterFactory {
+
+ RequestSubmitter createSubmitter(Properties properties, String[] headersAndValues);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/ComposeHttpStatusCodeChecker.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/ComposeHttpStatusCodeChecker.java
new file mode 100644
index 0000000..db9f4fe
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/ComposeHttpStatusCodeChecker.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.status;
+
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * An implementation of {@link HttpStatusCodeChecker} that checks Http Status code against include
+ * list, concrete value or {@link HttpResponseCodeType}.
+ */
+public class ComposeHttpStatusCodeChecker implements HttpStatusCodeChecker {
+
+ private static final Set<HttpStatusCodeChecker> DEFAULT_ERROR_CODES =
+ Set.of(
+ new TypeStatusCodeChecker(HttpResponseCodeType.CLIENT_ERROR),
+ new TypeStatusCodeChecker(HttpResponseCodeType.SERVER_ERROR));
+
+ private static final int MIN_HTTP_STATUS_CODE = 100;
+
+ /** Set of {@link HttpStatusCodeChecker} for include listed status codes. */
+ private final Set<IncludeListHttpStatusCodeChecker> includedCodes;
+
+ /**
+ * Set of {@link HttpStatusCodeChecker} that check status code againts value match or {@link
+ * HttpResponseCodeType} match.
+ */
+ private final Set<HttpStatusCodeChecker> errorCodes;
+
+ public ComposeHttpStatusCodeChecker(ComposeHttpStatusCodeCheckerConfig config) {
+ includedCodes = prepareIncludeList(config);
+ errorCodes = prepareErrorCodes(config);
+ }
+
+ /**
+ * Checks whether given status code is considered as a error code. This implementation checks if
+ * status code matches any single value mask like "404" or http type mask such as "4XX". Code
+ * that matches one of those masks and is not on an include list will be considered as error
+ * code.
+ *
+ * @param statusCode http status code to assess.
+ * @return true if status code is considered as error or false if not.
+ */
+ public boolean isErrorCode(int statusCode) {
+
+ Preconditions.checkArgument(
+ statusCode >= MIN_HTTP_STATUS_CODE,
+ String.format(
+ "Provided invalid Http status code %s,"
+ + " status code should be equal or bigger than %d.",
+ statusCode, MIN_HTTP_STATUS_CODE));
+
+ boolean isOnIncludeList =
+ includedCodes.stream().anyMatch(check -> check.isOnIncludeList(statusCode));
+
+ return !isOnIncludeList
+ && errorCodes.stream()
+ .anyMatch(
+ httpStatusCodeChecker ->
+ httpStatusCodeChecker.isErrorCode(statusCode));
+ }
+
+ private Set<HttpStatusCodeChecker> prepareErrorCodes(
+ ComposeHttpStatusCodeCheckerConfig config) {
+
+ Properties properties = config.getProperties();
+ String errorCodePrefix = config.getErrorCodePrefix();
+
+ String errorCodes = properties.getProperty(errorCodePrefix, "");
+
+ if (StringUtils.isNullOrWhitespaceOnly(errorCodes)) {
+ return DEFAULT_ERROR_CODES;
+ } else {
+ String[] splitCodes = errorCodes.split(HttpConnectorConfigConstants.PROP_DELIM);
+ return prepareErrorCodes(splitCodes);
+ }
+ }
+
+ /**
+ * Process given array of status codes and assign them to {@link
+ * SingleValueHttpStatusCodeChecker} for full codes such as 100, 404 etc. or to {@link
+ * TypeStatusCodeChecker} for codes that were constructed with "XX" mask
+ */
+ private Set<HttpStatusCodeChecker> prepareErrorCodes(String[] statusCodes) {
+
+ Set<HttpStatusCodeChecker> errorCodes = new HashSet<>();
+ for (String sCode : statusCodes) {
+ if (!StringUtils.isNullOrWhitespaceOnly(sCode)) {
+ String trimCode = sCode.toUpperCase().trim();
+ Preconditions.checkArgument(
+ trimCode.length() == 3,
+ "Status code should contain three characters. Provided [%s]",
+ trimCode);
+
+ // at this point we have trim, upper case 3 character status code.
+ if (isTypeCode(trimCode)) {
+ int code = Integer.parseInt(trimCode.replace("X", ""));
+ errorCodes.add(new TypeStatusCodeChecker(HttpResponseCodeType.getByCode(code)));
+ } else {
+ errorCodes.add(
+ new SingleValueHttpStatusCodeChecker(Integer.parseInt(trimCode)));
+ }
+ }
+ }
+ return (errorCodes.isEmpty()) ? DEFAULT_ERROR_CODES : errorCodes;
+ }
+
+ private Set<IncludeListHttpStatusCodeChecker> prepareIncludeList(
+ ComposeHttpStatusCodeCheckerConfig config) {
+
+ Properties properties = config.getProperties();
+ String includeListPrefix = config.getIncludeListPrefix();
+
+ return Arrays.stream(
+ properties
+ .getProperty(includeListPrefix, "")
+ .split(HttpConnectorConfigConstants.PROP_DELIM))
+ .filter(sCode -> !StringUtils.isNullOrWhitespaceOnly(sCode))
+ .map(String::trim)
+ .mapToInt(Integer::parseInt)
+ .mapToObj(IncludeListHttpStatusCodeChecker::new)
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * This method checks if "code" param matches "digit + XX" mask. This method expects that
+ * provided string will be 3 elements long, trim and upper case.
+ *
+ * @param code to check if it contains XX on second and third position. Parameter is expected to
+ * be 3 characters long, trim and uppercase.
+ * @return true if string matches "anything + XX" and false if not.
+ */
+ private boolean isTypeCode(final String code) {
+ return code.charAt(1) == 'X' && code.charAt(2) == 'X';
+ }
+
+ /** config. */
+ @Data
+ @Builder
+ @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+ public static class ComposeHttpStatusCodeCheckerConfig {
+
+ private final String includeListPrefix;
+
+ private final String errorCodePrefix;
+
+ private final Properties properties;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpCodesParser.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpCodesParser.java
new file mode 100644
index 0000000..dcb7dec
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpCodesParser.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.status;
+
+import org.apache.flink.util.ConfigurationException;
+
+import lombok.experimental.UtilityClass;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static java.lang.String.format;
+
+/** Utility class for parsing http codes. */
+@UtilityClass
+public class HttpCodesParser {
+
+ private static final Pattern CODE_GROUP_EXPRESSION = Pattern.compile("[1-5][xX]{2}");
+ private static final String DELIMITER = Pattern.quote(",");
+ private static final int HTTP_CODE_MIN = 100;
+ private static final int HTTP_CODE_MAX = 599;
+
+ public Set<Integer> parse(String codesExpression) throws ConfigurationException {
+ var includelist = new HashSet<Integer>();
+ var excludelist = new HashSet<Integer>();
+ for (var rawCode : codesExpression.split(DELIMITER)) {
+ var code = rawCode.trim();
+ if (code.isEmpty()) {
+ continue;
+ }
+ if (code.startsWith("!")) {
+ try {
+ excludelist.add(parseHttpCode(code.substring(1)));
+ continue;
+ } catch (NumberFormatException e) {
+ throw new ConfigurationException("Can not parse code " + code);
+ }
+ }
+ try {
+ includelist.add(parseHttpCode(code));
+ } catch (NumberFormatException e) {
+ if (CODE_GROUP_EXPRESSION.matcher(code).matches()) {
+ var firstGroupCode = Integer.parseInt(code.substring(0, 1)) * 100;
+ var groupCodes =
+ IntStream.range(firstGroupCode, firstGroupCode + 100)
+ .boxed()
+ .collect(Collectors.toList());
+ includelist.addAll(groupCodes);
+ } else {
+ throw new ConfigurationException("Can not parse code " + code);
+ }
+ }
+ }
+
+ includelist.removeAll(excludelist);
+ return Collections.unmodifiableSet(includelist);
+ }
+
+ private Integer parseHttpCode(String str) throws ConfigurationException {
+ var parsed = Integer.parseInt(str);
+ if (parsed < HTTP_CODE_MIN || parsed > HTTP_CODE_MAX) {
+ throw new ConfigurationException(format("Http code out of the range [%s]", parsed));
+ }
+ return parsed;
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/status/HttpResponseChecker.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseChecker.java
similarity index 56%
rename from src/main/java/com/getindata/connectors/http/internal/status/HttpResponseChecker.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseChecker.java
index 6b59c4b..5239f6c 100644
--- a/src/main/java/com/getindata/connectors/http/internal/status/HttpResponseChecker.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseChecker.java
@@ -1,13 +1,32 @@
-package com.getindata.connectors.http.internal.status;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.net.http.HttpResponse;
-import java.util.HashSet;
-import java.util.Set;
+package org.apache.flink.connector.http.status;
+
+import org.apache.flink.util.ConfigurationException;
import lombok.Getter;
import lombok.NonNull;
-import org.apache.flink.util.ConfigurationException;
+import java.net.http.HttpResponse;
+import java.util.HashSet;
+import java.util.Set;
+
+/** Http response checker. */
@Getter
public class HttpResponseChecker {
@@ -19,7 +38,8 @@ public class HttpResponseChecker {
this(HttpCodesParser.parse(successCodeExpr), HttpCodesParser.parse(temporalErrorCodeExpr));
}
- public HttpResponseChecker(@NonNull Set<Integer> successCodes, @NonNull Set<Integer> temporalErrorCodes)
+ public HttpResponseChecker(
+ @NonNull Set<Integer> successCodes, @NonNull Set<Integer> temporalErrorCodes)
throws ConfigurationException {
this.successCodes = successCodes;
this.temporalErrorCodes = temporalErrorCodes;
@@ -49,8 +69,10 @@ public class HttpResponseChecker {
var intersection = new HashSet<>(successCodes);
intersection.retainAll(temporalErrorCodes);
if (!intersection.isEmpty()) {
- throw new ConfigurationException("Http codes " + intersection +
- " can not be used as both success and retry codes");
+ throw new ConfigurationException(
+ "Http codes "
+ + intersection
+ + " can not be used as both success and retry codes");
}
}
}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseCodeType.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseCodeType.java
new file mode 100644
index 0000000..27f719e
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpResponseCodeType.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.status;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** This enum represents HTTP response code types, grouped by "hundreds" digit. */
+public enum HttpResponseCodeType {
+ INFO(1),
+ SUCCESS(2),
+ REDIRECTION(3),
+ CLIENT_ERROR(4),
+ SERVER_ERROR(5);
+
+ private static final Map<Integer, HttpResponseCodeType> map;
+
+ static {
+ map = new HashMap<>();
+ for (HttpResponseCodeType httpResponseCodeType : HttpResponseCodeType.values()) {
+ map.put(httpResponseCodeType.httpTypeCode, httpResponseCodeType);
+ }
+ }
+
+ private final int httpTypeCode;
+
+ HttpResponseCodeType(int httpTypeCode) {
+ this.httpTypeCode = httpTypeCode;
+ }
+
+ /**
+ * @param statusCode Http status code to get the {@link HttpResponseCodeType} instance for.
+ * @return a {@link HttpResponseCodeType} instance based on http type code, for example {@code
+ * HttpResponseCodeType.getByCode(1)} will return {@link HttpResponseCodeType#INFO} type.
+ */
+ public static HttpResponseCodeType getByCode(int statusCode) {
+ return map.get(statusCode);
+ }
+
+ /**
+ * @return a "hundreds" digit that represents given {@link HttpResponseCodeType} instance. For
+ * example {@code HttpResponseCodeType.INFO.getHttpTypeCode()} will return 1 since HTTP
+ * information repossess have status codes in range 100 - 199.
+ */
+ public int getHttpTypeCode() {
+ return this.httpTypeCode;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpStatusCodeChecker.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpStatusCodeChecker.java
new file mode 100644
index 0000000..6569b25
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/HttpStatusCodeChecker.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.status;
+
+/**
+ * Base interface for all classes that would validate HTTP status code whether it is an error or
+ * not.
+ */
+public interface HttpStatusCodeChecker {
+
+ /**
+ * Validates http status code wheter it is considered as error code. The logic for what status
+ * codes are considered as "errors" depends on the concreted implementation
+ *
+ * @param statusCode http status code to assess.
+ * @return true if statusCode is considered as Error and false if not.
+ */
+ boolean isErrorCode(int statusCode);
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/IncludeListHttpStatusCodeChecker.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/IncludeListHttpStatusCodeChecker.java
new file mode 100644
index 0000000..4d26d08
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/IncludeListHttpStatusCodeChecker.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.status;
+
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+
+/** Class that implements logic of an include list against single constant value. */
+@RequiredArgsConstructor
+@EqualsAndHashCode
+public class IncludeListHttpStatusCodeChecker {
+
+ private final int includeListCode;
+
+ /**
+ * Checks if given statusCode is considered as included.
+ *
+ * @param statusCode status code to check.
+ * @return true if given statusCode is on the include list and false if not.
+ */
+ public boolean isOnIncludeList(int statusCode) {
+ return includeListCode == statusCode;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/SingleValueHttpStatusCodeChecker.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/SingleValueHttpStatusCodeChecker.java
new file mode 100644
index 0000000..193b9ae
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/SingleValueHttpStatusCodeChecker.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.status;
+
+import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
+
+/**
+ * An implementation of {@link HttpStatusCodeChecker} that validates status code against constant
+ * value.
+ */
+@RequiredArgsConstructor
+@EqualsAndHashCode
+public class SingleValueHttpStatusCodeChecker implements HttpStatusCodeChecker {
+
+ /** A reference http status code to compare with. */
+ private final int errorCode;
+
+ /**
+ * Validates given statusCode against constant value.
+ *
+ * @param statusCode http status code to assess.
+ * @return true if status code is considered as error or false if not.
+ */
+ @Override
+ public boolean isErrorCode(int statusCode) {
+ return errorCode == statusCode;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/TypeStatusCodeChecker.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/TypeStatusCodeChecker.java
new file mode 100644
index 0000000..4e5311d
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/status/TypeStatusCodeChecker.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.status;
+
+import lombok.EqualsAndHashCode;
+
+/**
+ * Implementation of {@link HttpStatusCodeChecker} that verifies if given Http status code belongs
+ * to specific HTTP code type family. For example if it any of 100's 200's or 500's code.
+ */
+@EqualsAndHashCode
+public class TypeStatusCodeChecker implements HttpStatusCodeChecker {
+
+ /**
+ * First digit from HTTP status code that describes a type of code, for example 1 for all 100's,
+ * 5 for all 500's.
+ */
+ private final int httpTypeCode;
+
+ /**
+ * Creates TypeStatusCodeChecker for given {@link HttpResponseCodeType}.
+ *
+ * @param httpResponseCodeType {@link HttpResponseCodeType} for this {@link
+ * TypeStatusCodeChecker} instance.
+ */
+ public TypeStatusCodeChecker(HttpResponseCodeType httpResponseCodeType) {
+ this.httpTypeCode = httpResponseCodeType.getHttpTypeCode();
+ }
+
+ /**
+ * Checks whether given status code belongs to Http code status type. For example:
+ *
+ * <pre>{@code
+ * TypeStatusCodeChecker checker = new TypeStatusCodeChecker(5);
+ * checker.isErrorCode(505); <- will return true.
+ *
+ * }</pre>
+ *
+ * @param statusCode http status code to assess.
+ * @return true if status code is considered as error or false if not.
+ */
+ @Override
+ public boolean isErrorCode(int statusCode) {
+ return statusCode / 100 == httpTypeCode;
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/SerializationSchemaElementConverter.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/SerializationSchemaElementConverter.java
new file mode 100644
index 0000000..8d5e178
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/SerializationSchemaElementConverter.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table;
+
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.connector.sink2.Sink.InitContext;
+import org.apache.flink.api.connector.sink2.SinkWriter.Context;
+import org.apache.flink.connector.http.SchemaLifecycleAwareElementConverter;
+import org.apache.flink.connector.http.sink.HttpSinkRequestEntry;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.util.FlinkRuntimeException;
+
+/** Serialization Schema Element Converter. */
+public class SerializationSchemaElementConverter
+ implements SchemaLifecycleAwareElementConverter<RowData, HttpSinkRequestEntry> {
+
+ private final String insertMethod;
+
+ private final SerializationSchema<RowData> serializationSchema;
+
+ private boolean schemaOpened = false;
+
+ public SerializationSchemaElementConverter(
+ String insertMethod, SerializationSchema<RowData> serializationSchema) {
+
+ this.insertMethod = insertMethod;
+ this.serializationSchema = serializationSchema;
+ }
+
+ @Override
+ public void open(InitContext context) {
+ if (!schemaOpened) {
+ try {
+ serializationSchema.open(context.asSerializationSchemaInitializationContext());
+ schemaOpened = true;
+ } catch (Exception e) {
+ throw new FlinkRuntimeException("Failed to initialize serialization schema.", e);
+ }
+ }
+ }
+
+ @Override
+ public HttpSinkRequestEntry apply(RowData rowData, Context context) {
+ return new HttpSinkRequestEntry(insertMethod, serializationSchema.serialize(rowData));
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/AsyncHttpTableLookupFunction.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/AsyncHttpTableLookupFunction.java
new file mode 100644
index 0000000..1b1061d
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/AsyncHttpTableLookupFunction.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.utils.ThreadUtils;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.AsyncLookupFunction;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Collection;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/** Async Lookup function. */
+@Slf4j
+@RequiredArgsConstructor
+public class AsyncHttpTableLookupFunction extends AsyncLookupFunction {
+
+ private static final String PULLING_THREAD_POOL_SIZE = "8";
+
+ private static final String PUBLISHING_THREAD_POOL_SIZE = "4";
+
+ /**
+ * The {@link org.apache.flink.table.functions.TableFunction} we want to decorate with async
+ * framework.
+ */
+ private final HttpTableLookupFunction decorate;
+
+ /** Thread pool for polling data from Http endpoint. */
+ private transient ExecutorService pullingThreadPool;
+
+ /** Thread pool for publishing data to Flink. */
+ private transient ExecutorService publishingThreadPool;
+
+ @Override
+ public void open(FunctionContext context) throws Exception {
+ super.open(context);
+ decorate.open(context);
+
+ int pullingThreadPoolSize =
+ Integer.parseInt(
+ decorate.getOptions()
+ .getProperties()
+ .getProperty(
+ HttpConnectorConfigConstants
+ .LOOKUP_HTTP_PULING_THREAD_POOL_SIZE,
+ PULLING_THREAD_POOL_SIZE));
+
+ int publishingThreadPoolSize =
+ Integer.parseInt(
+ decorate.getOptions()
+ .getProperties()
+ .getProperty(
+ HttpConnectorConfigConstants
+ .LOOKUP_HTTP_RESPONSE_THREAD_POOL_SIZE,
+ PUBLISHING_THREAD_POOL_SIZE));
+
+ pullingThreadPool =
+ Executors.newFixedThreadPool(
+ pullingThreadPoolSize,
+ new ExecutorThreadFactory(
+ "http-async-lookup-worker", ThreadUtils.LOGGING_EXCEPTION_HANDLER));
+
+ publishingThreadPool =
+ Executors.newFixedThreadPool(
+ publishingThreadPoolSize,
+ new ExecutorThreadFactory(
+ "http-async-publishing-worker",
+ ThreadUtils.LOGGING_EXCEPTION_HANDLER));
+ }
+
+ @Override
+ public CompletableFuture<Collection<RowData>> asyncLookup(RowData keyRow) {
+ CompletableFuture<Collection<RowData>> future = new CompletableFuture<>();
+ future.completeAsync(() -> decorate.lookup(keyRow), pullingThreadPool);
+
+ // We don't want to use ForkJoinPool at all. We are using a different thread pool
+ // for publishing here intentionally to avoid thread starvation.
+ CompletableFuture<Collection<RowData>> resultFuture = new CompletableFuture<>();
+ future.whenCompleteAsync(
+ (result, throwable) -> {
+ if (throwable != null) {
+ log.error("Exception while processing Http Async request", throwable);
+ resultFuture.completeExceptionally(
+ new RuntimeException(
+ "Exception while processing Http Async request",
+ throwable));
+ } else {
+ resultFuture.complete(result);
+ }
+ },
+ publishingThreadPool);
+ return resultFuture;
+ }
+
+ public LookupRow getLookupRow() {
+ return decorate.getLookupRow();
+ }
+
+ public HttpLookupConfig getOptions() {
+ return decorate.getOptions();
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.publishingThreadPool.shutdownNow();
+ this.pullingThreadPool.shutdownNow();
+ super.close();
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/BodyBasedRequestFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/BodyBasedRequestFactory.java
new file mode 100644
index 0000000..dab7a38
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/BodyBasedRequestFactory.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.utils.uri.URIBuilder;
+
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Logger;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpRequest;
+import java.net.http.HttpRequest.BodyPublishers;
+import java.net.http.HttpRequest.Builder;
+import java.time.Duration;
+
+/**
+ * Implementation of {@link HttpRequestFactory} for REST calls that sends their parameters using
+ * request body or in the path.
+ */
+@Slf4j
+public class BodyBasedRequestFactory extends RequestFactoryBase {
+
+ private final String methodName;
+
+ public BodyBasedRequestFactory(
+ String methodName,
+ LookupQueryCreator lookupQueryCreator,
+ HeaderPreprocessor headerPreprocessor,
+ HttpLookupConfig options) {
+
+ super(lookupQueryCreator, headerPreprocessor, options);
+ this.methodName = methodName.toUpperCase();
+ }
+
+ /**
+ * Method for preparing {@link Builder} for REST request that sends their parameters in request
+ * body, for example PUT or POST methods.
+ *
+ * @param lookupQueryInfo lookup query info used for request body.
+ * @return {@link Builder} for given lookupQuery.
+ */
+ @Override
+ protected Builder setUpRequestMethod(LookupQueryInfo lookupQueryInfo) {
+ return HttpRequest.newBuilder()
+ .uri(constructUri(lookupQueryInfo))
+ .method(methodName, BodyPublishers.ofString(lookupQueryInfo.getLookupQuery()))
+ .timeout(Duration.ofSeconds(this.httpRequestTimeOutSeconds));
+ }
+
+ @Override
+ protected Logger getLogger() {
+ return log;
+ }
+
+ URI constructUri(LookupQueryInfo lookupQueryInfo) {
+ StringBuilder resolvedUrl = new StringBuilder(baseUrl);
+ if (lookupQueryInfo.hasBodyBasedUrlQueryParameters()) {
+ resolvedUrl
+ .append(baseUrl.contains("?") ? "&" : "?")
+ .append(lookupQueryInfo.getBodyBasedUrlQueryParameters());
+ }
+ resolvedUrl = resolvePathParameters(lookupQueryInfo, resolvedUrl);
+
+ try {
+ return new URIBuilder(resolvedUrl.toString()).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/GetRequestFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/GetRequestFactory.java
new file mode 100644
index 0000000..694f2f5
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/GetRequestFactory.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.utils.uri.URIBuilder;
+
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Logger;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpRequest;
+import java.net.http.HttpRequest.Builder;
+import java.time.Duration;
+
+/** Implementation of {@link HttpRequestFactory} for GET REST calls. */
+@Slf4j
+public class GetRequestFactory extends RequestFactoryBase {
+
+ public GetRequestFactory(
+ LookupQueryCreator lookupQueryCreator,
+ HeaderPreprocessor headerPreprocessor,
+ HttpLookupConfig options) {
+
+ super(lookupQueryCreator, headerPreprocessor, options);
+ }
+
+ @Override
+ protected Logger getLogger() {
+ return log;
+ }
+
+ /**
+ * Method for preparing {@link Builder} for REST GET request. Where lookupQueryInfo is used as
+ * query parameters for GET requests. for example:
+ *
+ * <pre>
+ * http:localhost:8080/service?id=1
+ * </pre>
+ *
+ * <p>or as payload for body-based requests with optional parameters, for example:
+ *
+ * <pre>
+ * http:localhost:8080/service?id=1
+ * body payload: { "uid": 2 }
+ * </pre>
+ *
+ * @param lookupQueryInfo lookup query info used for request query parameters.
+ * @return {@link Builder} for given GET lookupQuery
+ */
+ @Override
+ protected Builder setUpRequestMethod(LookupQueryInfo lookupQueryInfo) {
+ return HttpRequest.newBuilder()
+ .uri(constructGetUri(lookupQueryInfo))
+ .GET()
+ .timeout(Duration.ofSeconds(this.httpRequestTimeOutSeconds));
+ }
+
+ URI constructGetUri(LookupQueryInfo lookupQueryInfo) {
+ StringBuilder resolvedUrl = new StringBuilder(baseUrl);
+ if (lookupQueryInfo.hasLookupQuery()) {
+ resolvedUrl
+ .append(baseUrl.contains("?") ? "&" : "?")
+ .append(lookupQueryInfo.getLookupQuery());
+ }
+ resolvedUrl = resolvePathParameters(lookupQueryInfo, resolvedUrl);
+ try {
+ return new URIBuilder(resolvedUrl.toString()).build();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConfig.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConfig.java
new file mode 100644
index 0000000..d641278
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConfig.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.http.HttpPostRequestCallback;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+import java.io.Serializable;
+import java.util.Properties;
+
+/** Http lookup config. */
+@Builder
+@Data
+@RequiredArgsConstructor
+public class HttpLookupConfig implements Serializable {
+
+ private final String lookupMethod;
+
+ private final String url;
+
+ @Builder.Default private final boolean useAsync = false;
+
+ @Builder.Default private final Properties properties = new Properties();
+
+ @Builder.Default private final ReadableConfig readableConfig = new Configuration();
+
+ private final HttpPostRequestCallback<HttpLookupSourceRequestEntry> httpPostRequestCallback;
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java
new file mode 100644
index 0000000..849e350
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupConnectorOptions.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.connector.http.retry.RetryStrategyType;
+
+import java.time.Duration;
+
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.LOOKUP_SOURCE_HEADER_USE_RAW;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.OIDC_AUTH_TOKEN_ENDPOINT_URL;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.OIDC_AUTH_TOKEN_EXPIRY_REDUCTION;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.OIDC_AUTH_TOKEN_REQUEST;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_CONNECTION_TIMEOUT;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_IGNORE_RESPONSE_CODES;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_LOOKUP_QUERY_CREATOR_IDENTIFIER;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_LOOKUP_REQUEST_CALLBACK_IDENTIFIER;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_PROXY_HOST;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_PROXY_PASSWORD;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_PROXY_PORT;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_PROXY_USERNAME;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_RETRY_EXP_DELAY_INITIAL_BACKOFF;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_RETRY_EXP_DELAY_MAX_BACKOFF;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_RETRY_EXP_DELAY_MULTIPLIER;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_RETRY_FIXED_DELAY_DELAY;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_RETRY_RETRY_CODES;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_RETRY_STRATEGY_TYPE;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.SOURCE_RETRY_SUCCESS_CODES;
+
+/** http lookup connector options. */
+public class HttpLookupConnectorOptions {
+
+ public static final ConfigOption<String> URL =
+ ConfigOptions.key("url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The HTTP endpoint URL.");
+
+ public static final ConfigOption<String> URL_ARGS =
+ ConfigOptions.key("url-args")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The arguments that should be used for HTTP GET Request.");
+
+ public static final ConfigOption<Boolean> ASYNC_POLLING =
+ ConfigOptions.key("asyncPolling")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Whether to use Sync and Async polling mechanism");
+
+ public static final ConfigOption<String> LOOKUP_METHOD =
+ ConfigOptions.key("lookup-method")
+ .stringType()
+ .defaultValue("GET")
+ .withDescription("Method used for REST executed by lookup connector.");
+
+ public static final ConfigOption<String> LOOKUP_QUERY_CREATOR_IDENTIFIER =
+ ConfigOptions.key(SOURCE_LOOKUP_QUERY_CREATOR_IDENTIFIER).stringType().noDefaultValue();
+
+ public static final ConfigOption<String> LOOKUP_REQUEST_FORMAT =
+ ConfigOptions.key("lookup-request.format").stringType().defaultValue("json");
+
+ public static final ConfigOption<Boolean> USE_RAW_AUTH_HEADER =
+ ConfigOptions.key(LOOKUP_SOURCE_HEADER_USE_RAW)
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Whether to use the raw value of Authorization header");
+
+ public static final ConfigOption<String> REQUEST_CALLBACK_IDENTIFIER =
+ ConfigOptions.key(SOURCE_LOOKUP_REQUEST_CALLBACK_IDENTIFIER)
+ .stringType()
+ .defaultValue(Slf4jHttpLookupPostRequestCallbackFactory.IDENTIFIER);
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_OIDC_AUTH_TOKEN_ENDPOINT_URL =
+ ConfigOptions.key(OIDC_AUTH_TOKEN_ENDPOINT_URL)
+ .stringType()
+ .noDefaultValue()
+ .withDescription("OIDC Token endpoint url.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST =
+ ConfigOptions.key(OIDC_AUTH_TOKEN_REQUEST)
+ .stringType()
+ .noDefaultValue()
+ .withDescription("OIDC token request.");
+
+ public static final ConfigOption<Duration> SOURCE_LOOKUP_OIDC_AUTH_TOKEN_EXPIRY_REDUCTION =
+ ConfigOptions.key(OIDC_AUTH_TOKEN_EXPIRY_REDUCTION)
+ .durationType()
+ .defaultValue(Duration.ofSeconds(1))
+ .withDescription(
+ "OIDC authorization access token expiry"
+ + " reduction as a Duration."
+ + " A new access token is obtained if the token"
+ + " is older than it's expiry time minus this value.");
+
+ public static final ConfigOption<Duration> SOURCE_LOOKUP_CONNECTION_TIMEOUT =
+ ConfigOptions.key(SOURCE_CONNECTION_TIMEOUT)
+ .durationType()
+ .noDefaultValue()
+ .withDescription("Http client connection timeout.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_PROXY_HOST =
+ ConfigOptions.key(SOURCE_PROXY_HOST)
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Http client proxy host.");
+
+ public static final ConfigOption<Integer> SOURCE_LOOKUP_PROXY_PORT =
+ ConfigOptions.key(SOURCE_PROXY_PORT)
+ .intType()
+ .noDefaultValue()
+ .withDescription("Http client proxy port.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_PROXY_USERNAME =
+ ConfigOptions.key(SOURCE_PROXY_USERNAME)
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Http client proxy username for authentication.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_PROXY_PASSWORD =
+ ConfigOptions.key(SOURCE_PROXY_PASSWORD)
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Http client proxy password for authentication.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_RETRY_STRATEGY =
+ ConfigOptions.key(SOURCE_RETRY_STRATEGY_TYPE)
+ .stringType()
+ .defaultValue(RetryStrategyType.FIXED_DELAY.getCode())
+ .withDescription(
+ "Auto retry strategy type: fixed-delay (default) or exponential-delay.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_HTTP_SUCCESS_CODES =
+ ConfigOptions.key(SOURCE_RETRY_SUCCESS_CODES)
+ .stringType()
+ .defaultValue("2XX")
+ .withDescription(
+ "Comma separated http codes considered as success response. "
+ + "Use [1-5]XX for groups and '!' character for excluding.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_HTTP_RETRY_CODES =
+ ConfigOptions.key(SOURCE_RETRY_RETRY_CODES)
+ .stringType()
+ .defaultValue("500,503,504")
+ .withDescription(
+ "Comma separated http codes considered as transient errors. "
+ + "Use [1-5]XX for groups and '!' character for excluding.");
+
+ public static final ConfigOption<Duration> SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY =
+ ConfigOptions.key(SOURCE_RETRY_FIXED_DELAY_DELAY)
+ .durationType()
+ .defaultValue(Duration.ofSeconds(1))
+ .withDescription("Fixed-delay interval between retries.");
+
+ public static final ConfigOption<Duration>
+ SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_INITIAL_BACKOFF =
+ ConfigOptions.key(SOURCE_RETRY_EXP_DELAY_INITIAL_BACKOFF)
+ .durationType()
+ .defaultValue(Duration.ofSeconds(1))
+ .withDescription("Exponential-delay initial delay.");
+
+ public static final ConfigOption<Duration> SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MAX_BACKOFF =
+ ConfigOptions.key(SOURCE_RETRY_EXP_DELAY_MAX_BACKOFF)
+ .durationType()
+ .defaultValue(Duration.ofMinutes(1))
+ .withDescription("Exponential-delay maximum delay.");
+
+ public static final ConfigOption<Double> SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MULTIPLIER =
+ ConfigOptions.key(SOURCE_RETRY_EXP_DELAY_MULTIPLIER)
+ .doubleType()
+ .defaultValue(1.5)
+ .withDescription("Exponential-delay multiplier.");
+
+ public static final ConfigOption<String> SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES =
+ ConfigOptions.key(SOURCE_IGNORE_RESPONSE_CODES)
+ .stringType()
+ .defaultValue("")
+ .withDescription(
+ "Comma separated http codes. Content for these responses will be ignored. "
+ + "Use [1-5]XX for groups and '!' character for excluding. "
+ + "Ignored responses togater with `"
+ + SOURCE_RETRY_SUCCESS_CODES
+ + "` are considered as successful.");
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupSourceRequestEntry.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupSourceRequestEntry.java
new file mode 100644
index 0000000..98f9f9d
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupSourceRequestEntry.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import lombok.Data;
+import lombok.ToString;
+
+import java.net.http.HttpRequest;
+
+/**
+ * Wrapper class around {@link HttpRequest} that contains information about an actual lookup request
+ * body or request parameters.
+ */
+@Data
+@ToString
+public class HttpLookupSourceRequestEntry {
+
+ /** Wrapped {@link HttpRequest} object. */
+ private final HttpRequest httpRequest;
+
+ /**
+ * This field represents lookup query. Depending on used REST request method, this field can
+ * represent a request body, for example a Json string when PUT/POST requests method was used,
+ * or it can represent a query parameters if GET method was used.
+ */
+ private final LookupQueryInfo lookupQueryInfo;
+
+ public HttpLookupSourceRequestEntry(HttpRequest httpRequest, LookupQueryInfo lookupQueryInfo) {
+ this.httpRequest = httpRequest;
+ this.lookupQueryInfo = lookupQueryInfo;
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSource.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSource.java
similarity index 59%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSource.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSource.java
index 129837c..810dd5d 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSource.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSource.java
@@ -1,12 +1,31 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
-import java.util.ArrayList;
-import java.util.List;
-import javax.annotation.Nullable;
-
-import lombok.extern.slf4j.Slf4j;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.LookupQueryCreatorFactory;
+import org.apache.flink.connector.http.clients.PollingClientFactory;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.table.lookup.querycreators.GenericGetQueryCreatorFactory;
+import org.apache.flink.connector.http.table.lookup.querycreators.GenericJsonQueryCreatorFactory;
+import org.apache.flink.connector.http.utils.HttpHeaderUtils;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.DataTypes.Field;
import org.apache.flink.table.connector.Projection;
@@ -15,7 +34,7 @@ import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.connector.source.LookupTableSource;
import org.apache.flink.table.connector.source.abilities.SupportsLimitPushDown;
import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown;
-import org.apache.flink.table.connector.source.lookup.AsyncLookupFunctionProvider ;
+import org.apache.flink.table.connector.source.lookup.AsyncLookupFunctionProvider;
import org.apache.flink.table.connector.source.lookup.LookupFunctionProvider;
import org.apache.flink.table.connector.source.lookup.PartialCachingAsyncLookupProvider;
import org.apache.flink.table.connector.source.lookup.PartialCachingLookupProvider;
@@ -30,19 +49,20 @@ import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.table.types.logical.RowType.RowField;
import org.apache.flink.table.types.logical.utils.LogicalTypeChecks;
-import com.getindata.connectors.http.LookupQueryCreator;
-import com.getindata.connectors.http.LookupQueryCreatorFactory;
-import com.getindata.connectors.http.internal.HeaderPreprocessor;
-import com.getindata.connectors.http.internal.PollingClientFactory;
-import com.getindata.connectors.http.internal.table.lookup.querycreators.GenericGetQueryCreatorFactory;
-import com.getindata.connectors.http.internal.table.lookup.querycreators.GenericJsonQueryCreatorFactory;
-import com.getindata.connectors.http.internal.utils.HttpHeaderUtils;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.*;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupTableSourceFactory.row;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.LOOKUP_QUERY_CREATOR_IDENTIFIER;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupTableSourceFactory.row;
+/** http lookyp table source. */
@Slf4j
public class HttpLookupTableSource
- implements LookupTableSource, SupportsProjectionPushDown, SupportsLimitPushDown {
+ implements LookupTableSource, SupportsProjectionPushDown, SupportsLimitPushDown {
private DataType physicalRowDataType;
@@ -51,8 +71,7 @@ public class HttpLookupTableSource
private final DynamicTableFactory.Context dynamicTableFactoryContext;
private final DecodingFormat<DeserializationSchema<RowData>> decodingFormat;
- @Nullable
- private final LookupCache cache;
+ @Nullable private final LookupCache cache;
public HttpLookupTableSource(
DataType physicalRowDataType,
@@ -79,46 +98,38 @@ public class HttpLookupTableSource
LookupRow lookupRow = extractLookupRow(lookupContext.getKeys());
DeserializationSchema<RowData> responseSchemaDecoder =
- decodingFormat.createRuntimeDecoder(lookupContext, physicalRowDataType);
+ decodingFormat.createRuntimeDecoder(lookupContext, physicalRowDataType);
LookupQueryCreatorFactory lookupQueryCreatorFactory =
- FactoryUtil.discoverFactory(
- this.dynamicTableFactoryContext.getClassLoader(),
- LookupQueryCreatorFactory.class,
- lookupConfig.getReadableConfig().getOptional(LOOKUP_QUERY_CREATOR_IDENTIFIER)
- .orElse(
- (lookupConfig.getLookupMethod().equalsIgnoreCase("GET") ?
- GenericGetQueryCreatorFactory.IDENTIFIER :
- GenericJsonQueryCreatorFactory.IDENTIFIER)
- )
- );
+ FactoryUtil.discoverFactory(
+ this.dynamicTableFactoryContext.getClassLoader(),
+ LookupQueryCreatorFactory.class,
+ lookupConfig
+ .getReadableConfig()
+ .getOptional(LOOKUP_QUERY_CREATOR_IDENTIFIER)
+ .orElse(
+ (lookupConfig.getLookupMethod().equalsIgnoreCase("GET")
+ ? GenericGetQueryCreatorFactory.IDENTIFIER
+ : GenericJsonQueryCreatorFactory.IDENTIFIER)));
ReadableConfig readableConfig = lookupConfig.getReadableConfig();
LookupQueryCreator lookupQueryCreator =
- lookupQueryCreatorFactory.createLookupQueryCreator(
- readableConfig,
- lookupRow,
- dynamicTableFactoryContext
- );
+ lookupQueryCreatorFactory.createLookupQueryCreator(
+ readableConfig, lookupRow, dynamicTableFactoryContext);
PollingClientFactory<RowData> pollingClientFactory =
- createPollingClientFactory(lookupQueryCreator, lookupConfig);
+ createPollingClientFactory(lookupQueryCreator, lookupConfig);
return getLookupRuntimeProvider(lookupRow, responseSchemaDecoder, pollingClientFactory);
}
- protected LookupRuntimeProvider getLookupRuntimeProvider(LookupRow lookupRow,
- DeserializationSchema<RowData>
- responseSchemaDecoder,
- PollingClientFactory<RowData>
- pollingClientFactory) {
+ protected LookupRuntimeProvider getLookupRuntimeProvider(
+ LookupRow lookupRow,
+ DeserializationSchema<RowData> responseSchemaDecoder,
+ PollingClientFactory<RowData> pollingClientFactory) {
HttpTableLookupFunction dataLookupFunction =
new HttpTableLookupFunction(
- pollingClientFactory,
- responseSchemaDecoder,
- lookupRow,
- lookupConfig
- );
+ pollingClientFactory, responseSchemaDecoder, lookupRow, lookupConfig);
if (lookupConfig.isUseAsync()) {
AsyncLookupFunction asyncLookupFunction =
new AsyncHttpTableLookupFunction(dataLookupFunction);
@@ -143,12 +154,11 @@ public class HttpLookupTableSource
@Override
public DynamicTableSource copy() {
return new HttpLookupTableSource(
- physicalRowDataType,
- lookupConfig,
- decodingFormat,
- dynamicTableFactoryContext,
- cache
- );
+ physicalRowDataType,
+ lookupConfig,
+ decodingFormat,
+ dynamicTableFactoryContext,
+ cache);
}
@Override
@@ -157,8 +167,7 @@ public class HttpLookupTableSource
}
@Override
- public void applyLimit(long limit) {
- }
+ public void applyLimit(long limit) {}
@Override
public boolean supportsNestedProjection() {
@@ -166,24 +175,18 @@ public class HttpLookupTableSource
}
private PollingClientFactory<RowData> createPollingClientFactory(
- LookupQueryCreator lookupQueryCreator,
- HttpLookupConfig lookupConfig) {
+ LookupQueryCreator lookupQueryCreator, HttpLookupConfig lookupConfig) {
- HeaderPreprocessor headerPreprocessor = HttpHeaderUtils.createHeaderPreprocessor(
- lookupConfig.getReadableConfig());
+ HeaderPreprocessor headerPreprocessor =
+ HttpHeaderUtils.createHeaderPreprocessor(lookupConfig.getReadableConfig());
String lookupMethod = lookupConfig.getLookupMethod();
- HttpRequestFactory requestFactory = (lookupMethod.equalsIgnoreCase("GET")) ?
- new GetRequestFactory(
- lookupQueryCreator,
- headerPreprocessor,
- lookupConfig) :
- new BodyBasedRequestFactory(
- lookupMethod,
- lookupQueryCreator,
- headerPreprocessor,
- lookupConfig
- );
+ HttpRequestFactory requestFactory =
+ (lookupMethod.equalsIgnoreCase("GET"))
+ ? new GetRequestFactory(
+ lookupQueryCreator, headerPreprocessor, lookupConfig)
+ : new BodyBasedRequestFactory(
+ lookupMethod, lookupQueryCreator, headerPreprocessor, lookupConfig);
log.info("requestFactory is " + requestFactory);
return new JavaNetHttpPollingClientFactory(requestFactory);
}
@@ -193,10 +196,10 @@ public class HttpLookupTableSource
LookupRow lookupRow = new LookupRow();
List<String> fieldNames =
- TableSourceHelper.getFieldNames(physicalRowDataType.getLogicalType());
+ TableSourceHelper.getFieldNames(physicalRowDataType.getLogicalType());
List<LogicalType> fieldTypes =
- LogicalTypeChecks.getFieldTypes(physicalRowDataType.getLogicalType());
+ LogicalTypeChecks.getFieldTypes(physicalRowDataType.getLogicalType());
List<Field> lookupDataTypes = new ArrayList<>();
List<DataType> physicalRowDataTypes = physicalRowDataType.getChildren();
@@ -216,15 +219,12 @@ public class HttpLookupTableSource
}
private LookupSchemaEntry<RowData> extractKeyColumn(
- String name,
- int parentIndex,
- LogicalType type) {
+ String name, int parentIndex, LogicalType type) {
if (type instanceof RowType) {
- RowTypeLookupSchemaEntry rowLookupEntry = new RowTypeLookupSchemaEntry(
- name,
- RowData.createFieldGetter(type, parentIndex)
- );
+ RowTypeLookupSchemaEntry rowLookupEntry =
+ new RowTypeLookupSchemaEntry(
+ name, RowData.createFieldGetter(type, parentIndex));
List<RowField> fields = ((RowType) type).getFields();
int index = 0;
for (RowField rowField : fields) {
@@ -233,9 +233,7 @@ public class HttpLookupTableSource
return rowLookupEntry;
} else {
return new RowDataSingleValueLookupSchemaEntry(
- name,
- RowData.createFieldGetter(type, parentIndex)
- );
+ name, RowData.createFieldGetter(type, parentIndex));
}
}
@@ -243,8 +241,9 @@ public class HttpLookupTableSource
LogicalType type1 = rowField.getType();
String name = rowField.getName();
if (type1 instanceof RowType) {
- RowTypeLookupSchemaEntry rowLookupEntry = new RowTypeLookupSchemaEntry(name,
- RowData.createFieldGetter(type1, parentIndex));
+ RowTypeLookupSchemaEntry rowLookupEntry =
+ new RowTypeLookupSchemaEntry(
+ name, RowData.createFieldGetter(type1, parentIndex));
int index = 0;
List<RowField> rowFields = ((RowType) type1).getFields();
for (RowField rowField1 : rowFields) {
@@ -252,8 +251,8 @@ public class HttpLookupTableSource
}
return rowLookupEntry;
} else {
- return new RowDataSingleValueLookupSchemaEntry(name,
- RowData.createFieldGetter(type1, parentIndex));
+ return new RowDataSingleValueLookupSchemaEntry(
+ name, RowData.createFieldGetter(type1, parentIndex));
}
}
}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java
new file mode 100644
index 0000000..f9b07ab
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpLookupTableSourceFactory.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.http.HttpPostRequestCallbackFactory;
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.utils.ConfigUtils;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.DataTypes.Field;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.connector.source.lookup.LookupOptions;
+import org.apache.flink.table.connector.source.lookup.cache.DefaultLookupCache;
+import org.apache.flink.table.connector.source.lookup.cache.LookupCache;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DeserializationFormatFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.types.DataType;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.ASYNC_POLLING;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.LOOKUP_METHOD;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.LOOKUP_REQUEST_FORMAT;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.REQUEST_CALLBACK_IDENTIFIER;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_CONNECTION_TIMEOUT;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_RETRY_CODES;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_SUCCESS_CODES;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_OIDC_AUTH_TOKEN_ENDPOINT_URL;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_OIDC_AUTH_TOKEN_EXPIRY_REDUCTION;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_PROXY_HOST;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_PROXY_PASSWORD;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_PROXY_PORT;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_PROXY_USERNAME;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_INITIAL_BACKOFF;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MAX_BACKOFF;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MULTIPLIER;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_RETRY_STRATEGY;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.URL;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.URL_ARGS;
+import static org.apache.flink.table.api.DataTypes.FIELD;
+import static org.apache.flink.table.types.utils.DataTypeUtils.removeTimeAttribute;
+
+/** Http Lookup Table Source Factory. */
+public class HttpLookupTableSourceFactory implements DynamicTableSourceFactory {
+
+ private static DataTypes.Field columnToField(Column column) {
+ return FIELD(
+ column.getName(),
+ // only a column in a schema should have a time attribute,
+ // a field should not propagate the attribute because it might be used in a
+ // completely different context
+ removeTimeAttribute(column.getDataType()));
+ }
+
+ public static DataType row(List<Field> fields) {
+ return DataTypes.ROW(fields.toArray(new Field[0]));
+ }
+
+ @Override
+ public DynamicTableSource createDynamicTableSource(Context dynamicTableContext) {
+ FactoryUtil.TableFactoryHelper helper =
+ FactoryUtil.createTableFactoryHelper(this, dynamicTableContext);
+
+ ReadableConfig readable = helper.getOptions();
+ helper.validateExcept(
+ // properties coming from org.apache.flink.table.api.config.ExecutionConfigOptions
+ "table.",
+ HttpConnectorConfigConstants.FLINK_CONNECTOR_HTTP,
+ LOOKUP_REQUEST_FORMAT.key());
+ validateHttpLookupSourceOptions(readable);
+
+ DecodingFormat<DeserializationSchema<RowData>> decodingFormat =
+ helper.discoverDecodingFormat(
+ DeserializationFormatFactory.class, FactoryUtil.FORMAT);
+
+ HttpLookupConfig lookupConfig = getHttpLookupOptions(dynamicTableContext, readable);
+
+ ResolvedSchema resolvedSchema = dynamicTableContext.getCatalogTable().getResolvedSchema();
+
+ DataType physicalRowDataType =
+ toRowDataType(resolvedSchema.getColumns(), Column::isPhysical);
+
+ return new HttpLookupTableSource(
+ physicalRowDataType,
+ lookupConfig,
+ decodingFormat,
+ dynamicTableContext,
+ getLookupCache(readable));
+ }
+
+ protected void validateHttpLookupSourceOptions(ReadableConfig tableOptions)
+ throws IllegalArgumentException {
+ // ensure that there is an OIDC token request if we have an OIDC token endpoint
+ tableOptions
+ .getOptional(SOURCE_LOOKUP_OIDC_AUTH_TOKEN_ENDPOINT_URL)
+ .ifPresent(
+ url -> {
+ if (tableOptions
+ .getOptional(SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST)
+ .isEmpty()) {
+ throw new IllegalArgumentException(
+ "Config option "
+ + SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST.key()
+ + " is required, if "
+ + SOURCE_LOOKUP_OIDC_AUTH_TOKEN_ENDPOINT_URL.key()
+ + " is configured.");
+ }
+ });
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return "rest-lookup";
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return Set.of(URL, FactoryUtil.FORMAT);
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return Set.of(
+ URL_ARGS,
+ ASYNC_POLLING,
+ LOOKUP_METHOD,
+ REQUEST_CALLBACK_IDENTIFIER,
+ LookupOptions.CACHE_TYPE,
+ LookupOptions.PARTIAL_CACHE_EXPIRE_AFTER_ACCESS,
+ LookupOptions.PARTIAL_CACHE_EXPIRE_AFTER_WRITE,
+ LookupOptions.PARTIAL_CACHE_MAX_ROWS,
+ LookupOptions.PARTIAL_CACHE_CACHE_MISSING_KEY,
+ SOURCE_LOOKUP_OIDC_AUTH_TOKEN_EXPIRY_REDUCTION,
+ SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST,
+ SOURCE_LOOKUP_OIDC_AUTH_TOKEN_ENDPOINT_URL,
+ LookupOptions.MAX_RETRIES,
+ SOURCE_LOOKUP_RETRY_STRATEGY,
+ SOURCE_LOOKUP_RETRY_FIXED_DELAY_DELAY,
+ SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_INITIAL_BACKOFF,
+ SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MULTIPLIER,
+ SOURCE_LOOKUP_RETRY_EXPONENTIAL_DELAY_MAX_BACKOFF,
+ SOURCE_LOOKUP_HTTP_SUCCESS_CODES,
+ SOURCE_LOOKUP_HTTP_RETRY_CODES,
+ SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES,
+ SOURCE_LOOKUP_PROXY_HOST,
+ SOURCE_LOOKUP_PROXY_PORT,
+ SOURCE_LOOKUP_PROXY_USERNAME,
+ SOURCE_LOOKUP_PROXY_PASSWORD,
+ SOURCE_LOOKUP_CONNECTION_TIMEOUT // TODO: add request timeout from properties
+ );
+ }
+
+ private HttpLookupConfig getHttpLookupOptions(Context context, ReadableConfig readableConfig) {
+
+ Properties httpConnectorProperties =
+ ConfigUtils.getHttpConnectorProperties(context.getCatalogTable().getOptions());
+
+ final HttpPostRequestCallbackFactory<HttpLookupSourceRequestEntry>
+ postRequestCallbackFactory =
+ FactoryUtil.discoverFactory(
+ context.getClassLoader(),
+ HttpPostRequestCallbackFactory.class,
+ readableConfig.get(REQUEST_CALLBACK_IDENTIFIER));
+
+ return HttpLookupConfig.builder()
+ .lookupMethod(readableConfig.get(LOOKUP_METHOD))
+ .url(readableConfig.get(URL))
+ .useAsync(readableConfig.get(ASYNC_POLLING))
+ .properties(httpConnectorProperties)
+ .readableConfig(readableConfig)
+ .httpPostRequestCallback(postRequestCallbackFactory.createHttpPostRequestCallback())
+ .build();
+ }
+
+ @Nullable
+ private LookupCache getLookupCache(ReadableConfig tableOptions) {
+ LookupCache cache = null;
+ // Do not support legacy cache options
+ if (tableOptions
+ .get(LookupOptions.CACHE_TYPE)
+ .equals(LookupOptions.LookupCacheType.PARTIAL)) {
+ cache = DefaultLookupCache.fromConfig(tableOptions);
+ }
+ return cache;
+ }
+
+ // TODO verify this since we are on 1.15 now.
+ // Backport from Flink 1.15-Master
+ private DataType toRowDataType(List<Column> columns, Predicate<Column> columnPredicate) {
+ return columns.stream()
+ .filter(columnPredicate)
+ .map(HttpLookupTableSourceFactory::columnToField)
+ .collect(
+ Collectors.collectingAndThen(
+ Collectors.toList(), HttpLookupTableSourceFactory::row))
+ // the row should never be null
+ .notNull();
+ }
+
+ // Backport End
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpRequestFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpRequestFactory.java
new file mode 100644
index 0000000..4f22245
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpRequestFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.table.data.RowData;
+
+import java.io.Serializable;
+import java.net.http.HttpRequest;
+
+/** Factory for creating {@link HttpRequest} objects for Rest clients. */
+public interface HttpRequestFactory extends Serializable {
+
+ /**
+ * Creates a {@link HttpRequest} from given {@link RowData}.
+ *
+ * @param lookupRow {@link RowData} object used for building http request.
+ * @return {@link HttpRequest} created from {@link RowData}
+ */
+ HttpLookupSourceRequestEntry buildLookupRequest(RowData lookupRow);
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpTableLookupFunction.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpTableLookupFunction.java
similarity index 59%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpTableLookupFunction.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpTableLookupFunction.java
index 9c87ff4..d88e1a6 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpTableLookupFunction.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/HttpTableLookupFunction.java
@@ -1,21 +1,39 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.connector.http.clients.PollingClient;
+import org.apache.flink.connector.http.clients.PollingClientFactory;
+import org.apache.flink.connector.http.utils.SerializationSchemaUtils;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.functions.FunctionContext;
import org.apache.flink.table.functions.LookupFunction;
-import com.getindata.connectors.http.internal.PollingClient;
-import com.getindata.connectors.http.internal.PollingClientFactory;
-import com.getindata.connectors.http.internal.utils.SerializationSchemaUtils;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+/** lookup function. */
@Slf4j
public class HttpTableLookupFunction extends LookupFunction {
@@ -50,12 +68,11 @@ public class HttpTableLookupFunction extends LookupFunction {
@Override
public void open(FunctionContext context) throws Exception {
this.responseSchemaDecoder.open(
- SerializationSchemaUtils
- .createDeserializationInitContext(HttpTableLookupFunction.class));
+ SerializationSchemaUtils.createDeserializationInitContext(
+ HttpTableLookupFunction.class));
this.localHttpCallCounter = new AtomicInteger(0);
- this.client = pollingClientFactory
- .createPollClient(options, responseSchemaDecoder);
+ this.client = pollingClientFactory.createPollClient(options, responseSchemaDecoder);
context.getMetricGroup()
.gauge("http-table-lookup-call-counter", () -> localHttpCallCounter.intValue());
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClient.java
similarity index 58%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClient.java
index 956b2f0..4e904e8 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/JavaNetHttpPollingClient.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClient.java
@@ -1,4 +1,44 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.http.HttpPostRequestCallback;
+import org.apache.flink.connector.http.clients.PollingClient;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.retry.HttpClientWithRetry;
+import org.apache.flink.connector.http.retry.RetryConfigProvider;
+import org.apache.flink.connector.http.status.HttpCodesParser;
+import org.apache.flink.connector.http.status.HttpResponseChecker;
+import org.apache.flink.connector.http.utils.HttpHeaderUtils;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.util.ConfigurationException;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.NullNode;
+
+import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.net.http.HttpClient;
@@ -15,36 +55,15 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.NullNode;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.functions.FunctionContext;
-import org.apache.flink.util.ConfigurationException;
-import org.apache.flink.util.StringUtils;
-
-import com.getindata.connectors.http.HttpPostRequestCallback;
-import com.getindata.connectors.http.internal.HeaderPreprocessor;
-import com.getindata.connectors.http.internal.PollingClient;
-import com.getindata.connectors.http.internal.retry.HttpClientWithRetry;
-import com.getindata.connectors.http.internal.retry.RetryConfigProvider;
-import com.getindata.connectors.http.internal.status.HttpCodesParser;
-import com.getindata.connectors.http.internal.status.HttpResponseChecker;
-import com.getindata.connectors.http.internal.utils.HttpHeaderUtils;
-import static com.getindata.connectors.http.internal.config.HttpConnectorConfigConstants.RESULT_TYPE;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_RETRY_CODES;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_SUCCESS_CODES;
-import static com.getindata.connectors.http.internal.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST;
+import static org.apache.flink.connector.http.config.HttpConnectorConfigConstants.RESULT_TYPE;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_RETRY_CODES;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_HTTP_SUCCESS_CODES;
+import static org.apache.flink.connector.http.table.lookup.HttpLookupConnectorOptions.SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST;
/**
- * An implementation of {@link PollingClient} that uses Java 11's {@link HttpClient}.
- * This implementation supports HTTP traffic only.
+ * An implementation of {@link PollingClient} that uses Java 11's {@link HttpClient}. This
+ * implementation supports HTTP traffic only.
*/
@Slf4j
public class JavaNetHttpPollingClient implements PollingClient<RowData> {
@@ -64,7 +83,8 @@ public class JavaNetHttpPollingClient implements PollingClient<RowData> {
HttpClient httpClient,
DeserializationSchema<RowData> responseBodyDecoder,
HttpLookupConfig options,
- HttpRequestFactory requestFactory) throws ConfigurationException {
+ HttpRequestFactory requestFactory)
+ throws ConfigurationException {
this.responseBodyDecoder = responseBodyDecoder;
this.requestFactory = requestFactory;
@@ -74,24 +94,25 @@ public class JavaNetHttpPollingClient implements PollingClient<RowData> {
var config = options.getReadableConfig();
- this.ignoredErrorCodes = HttpCodesParser.parse(config.get(SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES));
+ this.ignoredErrorCodes =
+ HttpCodesParser.parse(config.get(SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES));
var errorCodes = HttpCodesParser.parse(config.get(SOURCE_LOOKUP_HTTP_RETRY_CODES));
var successCodes = new HashSet<Integer>();
successCodes.addAll(HttpCodesParser.parse(config.get(SOURCE_LOOKUP_HTTP_SUCCESS_CODES)));
successCodes.addAll(ignoredErrorCodes);
- this.httpClient = HttpClientWithRetry.builder()
- .httpClient(httpClient)
- .retryConfig(RetryConfigProvider.create(config))
- .responseChecker(new HttpResponseChecker(successCodes, errorCodes))
- .build();
+ this.httpClient =
+ HttpClientWithRetry.builder()
+ .httpClient(httpClient)
+ .retryConfig(RetryConfigProvider.create(config))
+ .responseChecker(new HttpResponseChecker(successCodes, errorCodes))
+ .build();
}
public void open(FunctionContext context) {
httpClient.registerMetrics(context.getMetricGroup());
}
-
@Override
public Collection<RowData> pull(RowData lookupRow) {
if (lookupRow == null) {
@@ -108,28 +129,31 @@ public class JavaNetHttpPollingClient implements PollingClient<RowData> {
private Collection<RowData> queryAndProcess(RowData lookupData) throws Exception {
var request = requestFactory.buildLookupRequest(lookupData);
- var oidcProcessor = HttpHeaderUtils.createOIDCHeaderPreprocessor(options.getReadableConfig());
- var response = httpClient.send(
- () -> updateHttpRequestIfRequired(request, oidcProcessor), BodyHandlers.ofString());
+ var oidcProcessor =
+ HttpHeaderUtils.createOIDCHeaderPreprocessor(options.getReadableConfig());
+ var response =
+ httpClient.send(
+ () -> updateHttpRequestIfRequired(request, oidcProcessor),
+ BodyHandlers.ofString());
return processHttpResponse(response, request);
}
/**
* If using OIDC, update the http request using the oidc header pre processor to supply the
* authentication header, with a short lived bearer token.
+ *
* @param request http reauest to amend
* @param oidcHeaderPreProcessor OIDC header pre processor
* @return http request, which for OIDC will have the bearer token as the authentication header
*/
- protected HttpRequest updateHttpRequestIfRequired(HttpLookupSourceRequestEntry request,
- HeaderPreprocessor oidcHeaderPreProcessor) {
+ protected HttpRequest updateHttpRequestIfRequired(
+ HttpLookupSourceRequestEntry request, HeaderPreprocessor oidcHeaderPreProcessor) {
// We need to check the config and if required amend the value of the
// authentication header to the short lived bearer token
HttpRequest httpRequest = request.getHttpRequest();
ReadableConfig readableConfig = options.getReadableConfig();
if (oidcHeaderPreProcessor != null) {
- HttpRequest.Builder builder = HttpRequest.newBuilder()
- .uri(httpRequest.uri());
+ HttpRequest.Builder builder = HttpRequest.newBuilder().uri(httpRequest.uri());
if (httpRequest.timeout().isPresent()) {
builder.timeout(httpRequest.timeout().get());
}
@@ -140,19 +164,20 @@ public class JavaNetHttpPollingClient implements PollingClient<RowData> {
}
Map<String, String> headerMap = new HashMap<>();
if (httpRequest.headers() != null && !httpRequest.headers().map().isEmpty()) {
- for (Map.Entry<String, List<String>> header
- :httpRequest.headers().map().entrySet()) {
- List<String> values = header.getValue();
+ for (Map.Entry<String, List<String>> header :
+ httpRequest.headers().map().entrySet()) {
+ List<String> values = header.getValue();
if (values.size() == 1) {
headerMap.put(header.getKey(), header.getValue().get(0));
}
// the existing design does not handle multiple values for headers
}
}
- Optional<String> oidcTokenRequest = readableConfig
- .getOptional(SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST);
- String bearerToken = oidcHeaderPreProcessor.preprocessValueForHeader(
- HttpHeaderUtils.AUTHORIZATION, oidcTokenRequest.get());
+ Optional<String> oidcTokenRequest =
+ readableConfig.getOptional(SOURCE_LOOKUP_OIDC_AUTH_TOKEN_REQUEST);
+ String bearerToken =
+ oidcHeaderPreProcessor.preprocessValueForHeader(
+ HttpHeaderUtils.AUTHORIZATION, oidcTokenRequest.get());
headerMap.put(HttpHeaderUtils.AUTHORIZATION, bearerToken);
String[] headerAndValueArray = HttpHeaderUtils.toHeaderAndValueArray(headerMap);
builder.headers(headerAndValueArray);
@@ -162,15 +187,17 @@ public class JavaNetHttpPollingClient implements PollingClient<RowData> {
}
private Collection<RowData> processHttpResponse(
- HttpResponse<String> response,
- HttpLookupSourceRequestEntry request) throws IOException {
+ HttpResponse<String> response, HttpLookupSourceRequestEntry request)
+ throws IOException {
this.httpPostRequestCallback.call(response, request, "endpoint", Collections.emptyMap());
var responseBody = response.body();
- log.debug("Received status code [{}] for RestTableSource request with Server response body [{}] ",
- response.statusCode(), responseBody);
+ log.debug(
+ "Received status code [{}] for RestTableSource request with Server response body [{}] ",
+ response.statusCode(),
+ responseBody);
if (StringUtils.isNullOrWhitespaceOnly(responseBody) || ignoreResponse(response)) {
return Collections.emptyList();
@@ -186,32 +213,30 @@ public class JavaNetHttpPollingClient implements PollingClient<RowData> {
private Collection<RowData> deserialize(String responseBody) throws IOException {
byte[] rawBytes = responseBody.getBytes();
String resultType =
- options.getProperties().getProperty(RESULT_TYPE, RESULT_TYPE_SINGLE_VALUE);
+ options.getProperties().getProperty(RESULT_TYPE, RESULT_TYPE_SINGLE_VALUE);
if (resultType.equals(RESULT_TYPE_SINGLE_VALUE)) {
return deserializeSingleValue(rawBytes);
} else if (resultType.equals(RESULT_TYPE_ARRAY)) {
return deserializeArray(rawBytes);
} else {
throw new IllegalStateException(
- String.format("Unknown lookup source result type '%s'.", resultType));
+ String.format("Unknown lookup source result type '%s'.", resultType));
}
}
private List<RowData> deserializeSingleValue(byte[] rawBytes) throws IOException {
return Optional.ofNullable(responseBodyDecoder.deserialize(rawBytes))
- .map(Collections::singletonList)
- .orElse(Collections.emptyList());
+ .map(Collections::singletonList)
+ .orElse(Collections.emptyList());
}
private List<RowData> deserializeArray(byte[] rawBytes) throws IOException {
- List<JsonNode> rawObjects =
- objectMapper.readValue(rawBytes, new TypeReference<>() {
- });
+ List<JsonNode> rawObjects = objectMapper.readValue(rawBytes, new TypeReference<>() {});
List<RowData> result = new ArrayList<>();
for (JsonNode rawObject : rawObjects) {
if (!(rawObject instanceof NullNode)) {
RowData deserialized =
- responseBodyDecoder.deserialize(rawObject.toString().getBytes());
+ responseBodyDecoder.deserialize(rawObject.toString().getBytes());
// deserialize() returns null if deserialization fails
if (deserialized != null) {
result.add(deserialized);
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientFactory.java
new file mode 100644
index 0000000..e002532
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/JavaNetHttpPollingClientFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.connector.http.clients.PollingClientFactory;
+import org.apache.flink.connector.http.utils.JavaNetHttpClientFactory;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.util.ConfigurationException;
+
+import java.net.http.HttpClient;
+
+/** JavaNetHttpPollingClientFactory. */
+public class JavaNetHttpPollingClientFactory implements PollingClientFactory<RowData> {
+
+ private final HttpRequestFactory requestFactory;
+
+ public JavaNetHttpPollingClientFactory(HttpRequestFactory requestFactory) {
+ this.requestFactory = requestFactory;
+ }
+
+ @Override
+ public JavaNetHttpPollingClient createPollClient(
+ HttpLookupConfig options, DeserializationSchema<RowData> schemaDecoder)
+ throws ConfigurationException {
+
+ HttpClient httpClient = JavaNetHttpClientFactory.createClient(options);
+
+ return new JavaNetHttpPollingClient(httpClient, schemaDecoder, options, requestFactory);
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupQueryInfo.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupQueryInfo.java
new file mode 100644
index 0000000..c314e32
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupQueryInfo.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.connector.http.utils.uri.NameValuePair;
+import org.apache.flink.connector.http.utils.uri.URLEncodedUtils;
+
+import lombok.Getter;
+import lombok.ToString;
+
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Holds the lookup query for an HTTP request. The {@code lookupQuery} either contain the query
+ * parameters for a GET operation or the payload of a body-based request. The {@code
+ * bodyBasedUrlQueryParams} contains the optional query parameters of a body-based request in
+ * addition to its payload supplied with {@code lookupQuery}.
+ */
+@ToString
+public class LookupQueryInfo implements Serializable {
+ @Getter private final String lookupQuery;
+
+ private final Map<String, String> bodyBasedUrlQueryParams;
+
+ private final Map<String, String> pathBasedUrlParams;
+
+ public LookupQueryInfo(String lookupQuery) {
+ this(lookupQuery, null, null);
+ }
+
+ public LookupQueryInfo(
+ String lookupQuery,
+ Map<String, String> bodyBasedUrlQueryParams,
+ Map<String, String> pathBasedUrlParams) {
+ this.lookupQuery = lookupQuery == null ? "" : lookupQuery;
+ this.bodyBasedUrlQueryParams =
+ bodyBasedUrlQueryParams == null ? Collections.emptyMap() : bodyBasedUrlQueryParams;
+ this.pathBasedUrlParams =
+ pathBasedUrlParams == null ? Collections.emptyMap() : pathBasedUrlParams;
+ }
+
+ public String getBodyBasedUrlQueryParameters() {
+ return URLEncodedUtils.format(
+ bodyBasedUrlQueryParams.entrySet().stream()
+ // sort the map by key to ensure there is a reliable order for unit tests
+ .sorted(Map.Entry.comparingByKey())
+ .map(entry -> new NameValuePair(entry.getKey(), entry.getValue()))
+ .collect(Collectors.toList()),
+ StandardCharsets.UTF_8);
+ }
+
+ public Map<String, String> getPathBasedUrlParameters() {
+ return pathBasedUrlParams;
+ }
+
+ public boolean hasLookupQuery() {
+ return !lookupQuery.isBlank();
+ }
+
+ public boolean hasBodyBasedUrlQueryParameters() {
+ return !bodyBasedUrlQueryParams.isEmpty();
+ }
+
+ public boolean hasPathBasedUrlParameters() {
+ return !pathBasedUrlParams.isEmpty();
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/LookupRow.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupRow.java
similarity index 59%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/LookupRow.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupRow.java
index bdd3f04..c84e3eb 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/LookupRow.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupRow.java
@@ -1,27 +1,44 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
+package org.apache.flink.connector.http.table.lookup;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.ToString;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.connector.http.LookupArg;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.DataType;
-import com.getindata.connectors.http.LookupArg;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+/** lookup row. */
@ToString
public class LookupRow implements Serializable {
private final List<LookupSchemaEntry<RowData>> lookupEntries;
- @Getter
- @Setter
- private DataType lookupPhysicalRowDataType;
+ @Getter @Setter private DataType lookupPhysicalRowDataType;
public LookupRow() {
this.lookupEntries = new LinkedList<>();
@@ -32,7 +49,7 @@ public class LookupRow implements Serializable {
* {@link RowData} is converted to {@link LookupArg}.
*
* @param lookupDataRow A {@link RowData} to get the values from for {@code
- * LookupArg#getArgValue()}.
+ * LookupArg#getArgValue()}.
* @return Collection of {@link LookupArg} objects created from lookupDataRow.
*/
public Collection<LookupArg> convertToLookupArgs(RowData lookupDataRow) {
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupSchemaEntry.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupSchemaEntry.java
new file mode 100644
index 0000000..3769490
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/LookupSchemaEntry.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.connector.http.LookupArg;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Represents Lookup entry with its name and provides conversion method to collection of {@link
+ * LookupArg} elements.
+ *
+ * @param <T> type of lookupKeyRow used for converting to {@link LookupArg}.
+ */
+public interface LookupSchemaEntry<T> extends Serializable {
+
+ /** @return lookup Field name. */
+ String getFieldName();
+
+ /**
+ * Creates a collection of {@link LookupArg} elements from provided T lookupKeyRow.
+ *
+ * @param lookupKeyRow Element to get the values from for {@code LookupArg#getArgValue()}.
+ * @return Collection of {@link LookupArg} objects created from lookupKeyRow.
+ */
+ List<LookupArg> convertToLookupArg(T lookupKeyRow);
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RequestFactoryBase.java
similarity index 55%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RequestFactoryBase.java
index 7b82228..56ffbd1 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RequestFactoryBase.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RequestFactoryBase.java
@@ -1,43 +1,55 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.net.http.HttpRequest;
-import java.net.http.HttpRequest.Builder;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.stream.Collectors;
+package org.apache.flink.connector.http.table.lookup;
-import lombok.extern.slf4j.Slf4j;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.config.HttpConnectorConfigConstants;
+import org.apache.flink.connector.http.preprocessor.HeaderPreprocessor;
+import org.apache.flink.connector.http.utils.HttpHeaderUtils;
import org.apache.flink.table.data.RowData;
import org.apache.flink.util.FlinkRuntimeException;
+
+import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
-import com.getindata.connectors.http.LookupQueryCreator;
-import com.getindata.connectors.http.internal.HeaderPreprocessor;
-import com.getindata.connectors.http.internal.config.HttpConnectorConfigConstants;
-import com.getindata.connectors.http.internal.utils.HttpHeaderUtils;
+import java.net.http.HttpRequest;
+import java.net.http.HttpRequest.Builder;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.stream.Collectors;
-/**
- * Base class for {@link HttpRequest} factories.
- */
+/** Base class for {@link HttpRequest} factories. */
@Slf4j
public abstract class RequestFactoryBase implements HttpRequestFactory {
public static final String DEFAULT_REQUEST_TIMEOUT_SECONDS = "30";
- /**
- * Base url used for {@link HttpRequest} for example "http://localhost:8080"
- */
+ /** Base url used for {@link HttpRequest} for example "http://localhost:8080". */
protected final String baseUrl;
protected final LookupQueryCreator lookupQueryCreator;
protected final int httpRequestTimeOutSeconds;
- /**
- * HTTP headers that should be used for {@link HttpRequest} created by factory.
- */
+ /** HTTP headers that should be used for {@link HttpRequest} created by factory. */
private final String[] headersAndValues;
+
private final HttpLookupConfig options;
public RequestFactoryBase(
@@ -52,25 +64,25 @@ public abstract class RequestFactoryBase implements HttpRequestFactory {
// issues a network call to the authentication server. This code is driven for
// explain select. Explain should not issue network calls.
// We setup the OIDC authentication header at lookup query time.
- var headerMap = HttpHeaderUtils
- .prepareHeaderMap(
- HttpConnectorConfigConstants.LOOKUP_SOURCE_HEADER_PREFIX,
- options.getProperties(),
- headerPreprocessor
- );
+ var headerMap =
+ HttpHeaderUtils.prepareHeaderMap(
+ HttpConnectorConfigConstants.LOOKUP_SOURCE_HEADER_PREFIX,
+ options.getProperties(),
+ headerPreprocessor);
this.headersAndValues = HttpHeaderUtils.toHeaderAndValueArray(headerMap);
- log.debug("RequestFactoryBase headersAndValues: " +
- Arrays.stream(headersAndValues)
- .map(Object::toString)
- .collect(Collectors.joining(",")));
- this.httpRequestTimeOutSeconds = Integer.parseInt(
- options.getProperties().getProperty(
- HttpConnectorConfigConstants.LOOKUP_HTTP_TIMEOUT_SECONDS,
- DEFAULT_REQUEST_TIMEOUT_SECONDS
- )
- );
+ log.debug(
+ "RequestFactoryBase headersAndValues: "
+ + Arrays.stream(headersAndValues)
+ .map(Object::toString)
+ .collect(Collectors.joining(",")));
+ this.httpRequestTimeOutSeconds =
+ Integer.parseInt(
+ options.getProperties()
+ .getProperty(
+ HttpConnectorConfigConstants.LOOKUP_HTTP_TIMEOUT_SECONDS,
+ DEFAULT_REQUEST_TIMEOUT_SECONDS));
}
@Override
@@ -90,14 +102,15 @@ public abstract class RequestFactoryBase implements HttpRequestFactory {
protected abstract Logger getLogger();
/**
- * Method for preparing {@link HttpRequest.Builder} for concrete REST method.
+ * Method for preparing {@link Builder} for concrete REST method.
+ *
* @param lookupQuery lookup query used for request query parameters or body.
- * @return {@link HttpRequest.Builder} for given lookupQuery.
+ * @return {@link Builder} for given lookupQuery.
*/
protected abstract Builder setUpRequestMethod(LookupQueryInfo lookupQuery);
- protected static StringBuilder resolvePathParameters(LookupQueryInfo lookupQueryInfo,
- StringBuilder resolvedUrl) {
+ protected static StringBuilder resolvePathParameters(
+ LookupQueryInfo lookupQueryInfo, StringBuilder resolvedUrl) {
if (lookupQueryInfo.hasPathBasedUrlParameters()) {
for (Map.Entry<String, String> entry :
lookupQueryInfo.getPathBasedUrlParameters().entrySet()) {
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataLookupSchemaEntryBase.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataLookupSchemaEntryBase.java
new file mode 100644
index 0000000..03927c4
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataLookupSchemaEntryBase.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.table.data.RowData;
+
+/** Base implementation of {@link LookupSchemaEntry} for {@link RowData} type. */
+public abstract class RowDataLookupSchemaEntryBase<T> implements LookupSchemaEntry<RowData> {
+
+ /** Lookup field name represented by this instance. */
+ protected final String fieldName;
+
+ /** {@link RowData.FieldGetter} matching RowData type for field represented by this instance. */
+ protected final RowData.FieldGetter fieldGetter;
+
+ /**
+ * Creates new instance.
+ *
+ * @param fieldName field name that this instance represents, matching {@link RowData} column
+ * name.
+ * @param fieldGetter {@link RowData.FieldGetter} for data type matching {@link RowData} column
+ * type that this instance represents.
+ */
+ public RowDataLookupSchemaEntryBase(String fieldName, RowData.FieldGetter fieldGetter) {
+ this.fieldName = fieldName;
+ this.fieldGetter = fieldGetter;
+ }
+
+ public String getFieldName() {
+ return this.fieldName;
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RowDataSingleValueLookupSchemaEntry.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataSingleValueLookupSchemaEntry.java
similarity index 57%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/RowDataSingleValueLookupSchemaEntry.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataSingleValueLookupSchemaEntry.java
index e977ca7..31f31e7 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RowDataSingleValueLookupSchemaEntry.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowDataSingleValueLookupSchemaEntry.java
@@ -1,14 +1,31 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.util.Collections;
-import java.util.List;
+package org.apache.flink.connector.http.table.lookup;
-import lombok.extern.slf4j.Slf4j;
+import org.apache.flink.connector.http.LookupArg;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.RowData.FieldGetter;
import org.apache.flink.table.data.binary.BinaryStringData;
-import com.getindata.connectors.http.LookupArg;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Collections;
+import java.util.List;
/**
* Implementation of {@link LookupSchemaEntry} for {@link RowData} type that represents single
@@ -20,10 +37,10 @@ public class RowDataSingleValueLookupSchemaEntry extends RowDataLookupSchemaEntr
/**
* Creates new instance.
*
- * @param fieldName field name that this instance represents, matching {@link RowData} column
- * name.
+ * @param fieldName field name that this instance represents, matching {@link RowData} column
+ * name.
* @param fieldGetter {@link RowData.FieldGetter} for data type matching {@link RowData} column
- * type that this instance represents.
+ * type that this instance represents.
*/
public RowDataSingleValueLookupSchemaEntry(String fieldName, FieldGetter fieldGetter) {
super(fieldName, fieldGetter);
@@ -47,8 +64,7 @@ public class RowDataSingleValueLookupSchemaEntry extends RowDataLookupSchemaEntr
}
if (!(value instanceof BinaryStringData)) {
- log.debug("Unsupported Key Type {}. Trying simple toString().",
- value.getClass());
+ log.debug("Unsupported Key Type {}. Trying simple toString().", value.getClass());
}
return Collections.singletonList(new LookupArg(getFieldName(), value.toString()));
@@ -59,16 +75,19 @@ public class RowDataSingleValueLookupSchemaEntry extends RowDataLookupSchemaEntr
return fieldGetter.getFieldOrNull(lookupKeyRow);
} catch (ClassCastException e) {
throw new RuntimeException(
- "Class cast exception on field getter for field " + getFieldName(), e);
+ "Class cast exception on field getter for field " + getFieldName(), e);
}
}
@lombok.Generated
@Override
public String toString() {
- return "RowDataSingleValueLookupSchemaEntry{" +
- "fieldName='" + fieldName + '\'' +
- ", fieldGetter=" + fieldGetter +
- '}';
+ return "RowDataSingleValueLookupSchemaEntry{"
+ + "fieldName='"
+ + fieldName
+ + '\''
+ + ", fieldGetter="
+ + fieldGetter
+ + '}';
}
}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RowTypeLookupSchemaEntry.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowTypeLookupSchemaEntry.java
similarity index 63%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/RowTypeLookupSchemaEntry.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowTypeLookupSchemaEntry.java
index d9812bb..7276c93 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/RowTypeLookupSchemaEntry.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/RowTypeLookupSchemaEntry.java
@@ -1,13 +1,29 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
+package org.apache.flink.connector.http.table.lookup;
+import org.apache.flink.connector.http.LookupArg;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.RowData.FieldGetter;
-import com.getindata.connectors.http.LookupArg;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
/**
* Implementation of {@link LookupSchemaEntry} for {@link RowData} type that represents multiple
@@ -24,10 +40,10 @@ public class RowTypeLookupSchemaEntry extends RowDataLookupSchemaEntryBase<RowDa
/**
* Creates new instance.
*
- * @param fieldName field name that this instance represents, matching {@link RowData} column
- * name.
+ * @param fieldName field name that this instance represents, matching {@link RowData} column
+ * name.
* @param fieldGetter {@link RowData.FieldGetter} for data type matching {@link RowData} column
- * type that this instance represents.
+ * type that this instance represents.
*/
public RowTypeLookupSchemaEntry(String fieldName, FieldGetter fieldGetter) {
super(fieldName, fieldGetter);
@@ -73,10 +89,14 @@ public class RowTypeLookupSchemaEntry extends RowDataLookupSchemaEntryBase<RowDa
@lombok.Generated
@Override
public String toString() {
- return "RowTypeLookupSchemaEntry{" +
- "fieldName='" + fieldName + '\'' +
- ", fieldGetter=" + fieldGetter +
- ", keyColumns=" + keyColumns +
- '}';
+ return "RowTypeLookupSchemaEntry{"
+ + "fieldName='"
+ + fieldName
+ + '\''
+ + ", fieldGetter="
+ + fieldGetter
+ + ", keyColumns="
+ + keyColumns
+ + '}';
}
}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4JHttpLookupPostRequestCallback.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4JHttpLookupPostRequestCallback.java
new file mode 100644
index 0000000..3e66d9c
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4JHttpLookupPostRequestCallback.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.connector.http.HttpPostRequestCallback;
+import org.apache.flink.connector.http.utils.ConfigUtils;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.StringJoiner;
+
+/**
+ * A {@link HttpPostRequestCallback} that logs pairs of request and response as <i>INFO</i> level
+ * logs using <i>Slf4j</i>.
+ *
+ * <p>Serving as a default implementation of {@link HttpPostRequestCallback} for the {@link
+ * HttpLookupTableSource}.
+ */
+@Slf4j
+public class Slf4JHttpLookupPostRequestCallback
+ implements HttpPostRequestCallback<HttpLookupSourceRequestEntry> {
+
+ @Override
+ public void call(
+ HttpResponse<String> response,
+ HttpLookupSourceRequestEntry requestEntry,
+ String endpointUrl,
+ Map<String, String> headerMap) {
+
+ HttpRequest httpRequest = requestEntry.getHttpRequest();
+ StringJoiner headers = new StringJoiner(";");
+
+ for (Entry<String, List<String>> reqHeaders : httpRequest.headers().map().entrySet()) {
+ StringJoiner values = new StringJoiner(";");
+ for (String value : reqHeaders.getValue()) {
+ values.add(value);
+ }
+ String header = reqHeaders.getKey() + ": [" + values + "]";
+ headers.add(header);
+ }
+
+ if (response == null) {
+ log.warn("Null Http response for request " + httpRequest.uri().toString());
+
+ log.info(
+ "Got response for a request.\n Request:\n URL: {}\n "
+ + "Method: {}\n Headers: {}\n Params/Body: {}\nResponse: null",
+ httpRequest.uri().toString(),
+ httpRequest.method(),
+ headers,
+ requestEntry.getLookupQueryInfo());
+ } else {
+ log.info(
+ "Got response for a request.\n Request:\n URL: {}\n "
+ + "Method: {}\n Headers: {}\n Params/Body: {}\nResponse: {}\n Body: {}",
+ httpRequest.uri().toString(),
+ httpRequest.method(),
+ headers,
+ requestEntry.getLookupQueryInfo(),
+ response,
+ response.body().replaceAll(ConfigUtils.UNIVERSAL_NEW_LINE_REGEXP, ""));
+ }
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4jHttpLookupPostRequestCallbackFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4jHttpLookupPostRequestCallbackFactory.java
new file mode 100644
index 0000000..39e839e
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/Slf4jHttpLookupPostRequestCallbackFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.connector.http.HttpPostRequestCallback;
+import org.apache.flink.connector.http.HttpPostRequestCallbackFactory;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/** Factory for creating {@link Slf4JHttpLookupPostRequestCallback}. */
+public class Slf4jHttpLookupPostRequestCallbackFactory
+ implements HttpPostRequestCallbackFactory<HttpLookupSourceRequestEntry> {
+
+ public static final String IDENTIFIER = "slf4j-lookup-logger";
+
+ @Override
+ public HttpPostRequestCallback<HttpLookupSourceRequestEntry> createHttpPostRequestCallback() {
+ return new Slf4JHttpLookupPostRequestCallback();
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return new HashSet<>();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return new HashSet<>();
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/TableSourceHelper.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/TableSourceHelper.java
similarity index 64%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/TableSourceHelper.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/TableSourceHelper.java
index 3996fed..1fb6399 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/TableSourceHelper.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/TableSourceHelper.java
@@ -1,27 +1,47 @@
-package com.getindata.connectors.http.internal.table.lookup;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.util.Collections;
-import java.util.List;
+package org.apache.flink.connector.http.table.lookup;
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.LogicalTypeRoot;
import org.apache.flink.table.types.logical.utils.LogicalTypeChecks;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import java.util.Collections;
+import java.util.List;
+
import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.isCompositeType;
+/** table source helper. */
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public final class TableSourceHelper {
/**
* Returns the first-level field names for the provided {@link DataType}.
*
- * <p>Note: This method returns an empty list for every {@link DataType} that is not a
- * composite
+ * <p>Note: This method returns an empty list for every {@link DataType} that is not a composite
* type.
+ *
* @param type logical type
* @return List of field names
*/
@@ -37,6 +57,7 @@ public final class TableSourceHelper {
/**
* Builds {@link RowData} object based on provided list of values.
+ *
* @param values values to use as {@link RowData} column values.
* @return new {@link RowData} instance.
*/
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreator.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreator.java
new file mode 100644
index 0000000..131a6ef
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreator.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup.querycreators;
+
+import org.apache.flink.connector.http.LookupArg;
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.table.lookup.LookupQueryInfo;
+import org.apache.flink.connector.http.table.lookup.LookupRow;
+import org.apache.flink.table.data.RowData;
+
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+/**
+ * A {@link LookupQueryCreator} that prepares <a
+ * href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html#search-api-query-params-q">
+ * <code>q</code> parameter GET query</a> for ElasticSearch <i>Search API</i> using Lucene query
+ * string syntax (in first versions of the ElasticSearch called <a
+ * href="https://www.elastic.co/guide/en/elasticsearch/guide/current/search-lite.html">Search
+ * <i>Lite</i></a>).
+ */
+public class ElasticSearchLiteQueryCreator implements LookupQueryCreator {
+
+ private static final String ENCODED_SPACE = "%20";
+ private static final String ENCODED_QUOTATION_MARK = "%22";
+
+ private final LookupRow lookupRow;
+
+ public ElasticSearchLiteQueryCreator(LookupRow lookupRow) {
+ this.lookupRow = lookupRow;
+ }
+
+ private static String processLookupArg(LookupArg arg) {
+ return arg.getArgName()
+ + ":"
+ + ENCODED_QUOTATION_MARK
+ + arg.getArgValue()
+ + ENCODED_QUOTATION_MARK;
+ }
+
+ @Override
+ public LookupQueryInfo createLookupQuery(RowData lookupDataRow) {
+ Collection<LookupArg> lookupArgs = lookupRow.convertToLookupArgs(lookupDataRow);
+
+ var luceneQuery =
+ lookupArgs.stream()
+ .map(ElasticSearchLiteQueryCreator::processLookupArg)
+ .collect(Collectors.joining(ENCODED_SPACE + "AND" + ENCODED_SPACE));
+
+ String lookupQuery = luceneQuery.isEmpty() ? "" : ("q=" + luceneQuery);
+
+ return new LookupQueryInfo(lookupQuery);
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreatorFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreatorFactory.java
new file mode 100644
index 0000000..6b0e9c2
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/ElasticSearchLiteQueryCreatorFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup.querycreators;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.LookupQueryCreatorFactory;
+import org.apache.flink.connector.http.table.lookup.LookupRow;
+import org.apache.flink.table.factories.DynamicTableFactory;
+
+import java.util.Set;
+
+/** Factory for creating {@link ElasticSearchLiteQueryCreator}. */
+public class ElasticSearchLiteQueryCreatorFactory implements LookupQueryCreatorFactory {
+
+ public static final String IDENTIFIER = "elasticsearch-lite";
+
+ @Override
+ public LookupQueryCreator createLookupQueryCreator(
+ ReadableConfig readableConfig,
+ LookupRow lookupRow,
+ DynamicTableFactory.Context dynamicTableFactoryContext) {
+ return new ElasticSearchLiteQueryCreator(lookupRow);
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return Set.of();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return Set.of();
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreator.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreator.java
new file mode 100644
index 0000000..387a088
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup.querycreators;
+
+import org.apache.flink.connector.http.LookupArg;
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.table.lookup.LookupQueryInfo;
+import org.apache.flink.connector.http.table.lookup.LookupRow;
+import org.apache.flink.connector.http.utils.uri.NameValuePair;
+import org.apache.flink.connector.http.utils.uri.URLEncodedUtils;
+import org.apache.flink.table.data.RowData;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+/**
+ * A {@link LookupQueryCreator} that builds an "ordinary" GET query, i.e. adds <code>
+ * joinColumn1=value1&joinColumn2=value2&...</code> to the URI of the endpoint.
+ */
+public class GenericGetQueryCreator implements LookupQueryCreator {
+
+ private final LookupRow lookupRow;
+
+ public GenericGetQueryCreator(LookupRow lookupRow) {
+ this.lookupRow = lookupRow;
+ }
+
+ @Override
+ public LookupQueryInfo createLookupQuery(RowData lookupDataRow) {
+
+ Collection<LookupArg> lookupArgs = lookupRow.convertToLookupArgs(lookupDataRow);
+
+ String lookupQuery =
+ URLEncodedUtils.format(
+ lookupArgs.stream()
+ .map(arg -> new NameValuePair(arg.getArgName(), arg.getArgValue()))
+ .collect(Collectors.toList()),
+ StandardCharsets.UTF_8);
+
+ return new LookupQueryInfo(lookupQuery);
+ }
+}
diff --git a/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreatorFactory.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreatorFactory.java
new file mode 100644
index 0000000..2894ff1
--- /dev/null
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericGetQueryCreatorFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.http.table.lookup.querycreators;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.http.LookupQueryCreator;
+import org.apache.flink.connector.http.LookupQueryCreatorFactory;
+import org.apache.flink.connector.http.table.lookup.LookupRow;
+import org.apache.flink.table.factories.DynamicTableFactory;
+
+import java.util.Set;
+
+/** Factory for creating {@link GenericGetQueryCreator}. */
+public class GenericGetQueryCreatorFactory implements LookupQueryCreatorFactory {
+
+ public static final String IDENTIFIER = "generic-get-query";
+
+ @Override
+ public LookupQueryCreator createLookupQueryCreator(
+ ReadableConfig readableConfig,
+ LookupRow lookupRow,
+ DynamicTableFactory.Context dynamicTableFactoryContext) {
+ return new GenericGetQueryCreator(lookupRow);
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return Set.of();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return Set.of();
+ }
+}
diff --git a/src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java
similarity index 53%
rename from src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java
rename to flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java
index f67a25a..a5c1f6f 100644
--- a/src/main/java/com/getindata/connectors/http/internal/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java
+++ b/flink-connector-http/src/main/java/org/apache/flink/connector/http/table/lookup/querycreators/GenericJsonAndUrlQueryCreator.java
@@ -1,21 +1,29 @@
/*
- * © Copyright IBM Corp. 2025
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
-package com.getindata.connectors.http.internal.table.lookup.querycreators;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.util.*;
+package org.apache.flink.connector.http.table.lookup.querycreators;
-import lombok.extern.slf4j.Slf4j;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
... 27189 lines suppressed ...