You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tuweni.apache.org by to...@apache.org on 2021/06/28 21:53:31 UTC
[incubator-tuweni] branch main updated: Configure TODOs - move to
config
This is an automated email from the ASF dual-hosted git repository.
toulmean pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-tuweni.git
The following commit(s) were added to refs/heads/main by this push:
new 90ea925 Configure TODOs - move to config
new 8c8c4c6 Merge pull request #294 from atoulme/configure_todos
90ea925 is described below
commit 90ea92501f341a17f546238e6b447f2254b06dee
Author: Antoine Toulme <an...@lunar-ocean.com>
AuthorDate: Mon Jun 28 22:44:36 2021 +0200
Configure TODOs - move to config
---
.../kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt | 8 ++++----
.../org/apache/tuweni/eth/crawler/CrawlerConfig.kt | 17 +++++++++++++++--
.../org/apache/tuweni/eth/crawler/CrawlerRESTService.kt | 4 ++--
.../tuweni/eth/crawler/RelationalPeerRepository.kt | 6 ++++--
4 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt
index 444f8ea..c6fb3fb 100644
--- a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt
+++ b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt
@@ -93,7 +93,7 @@ class CrawlerApplication(
fun run(vertx: Vertx, config: CrawlerConfig) {
val dbConfig = HikariConfig()
- dbConfig.maximumPoolSize = 25 // TODO make this a config setting.
+ dbConfig.maximumPoolSize = config.jdbcConnections()
dbConfig.jdbcUrl = config.jdbcUrl()
val ds = HikariDataSource(dbConfig)
val flyway = Flyway.configure()
@@ -101,7 +101,7 @@ class CrawlerApplication(
.load()
flyway.migrate()
- val repo = RelationalPeerRepository(ds)
+ val repo = RelationalPeerRepository(ds, config.peerCacheExpiration(), config.clientIdsInterval(), coroutineContext)
logger.info("Initial bootnodes: ${config.bootNodes()}")
val scraper = Scraper(
@@ -156,7 +156,7 @@ class CrawlerApplication(
}
wireConnectionsRepository.addConnectionListener {
launch {
- delay(10 * 1000) // TODO this delay can be made configurable.
+ delay(config.rlpxDisconnectionDelay())
it.disconnect(DisconnectReason.CLIENT_QUITTING)
}
}
@@ -178,7 +178,7 @@ class CrawlerApplication(
}
}
val restService =
- CrawlerRESTService(port = config.restPort(), networkInterface = config.restNetworkInterface(), repository = repo)
+ CrawlerRESTService(port = config.restPort(), networkInterface = config.restNetworkInterface(), repository = repo, maxRequestsPerSec = config.maxRequestsPerSec())
val refreshLoop = AtomicBoolean(true)
val ethstatsDataRepository = EthstatsDataRepository(ds)
val ethstatsServer = EthStatsServer(
diff --git a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt
index ae21c87..1066272 100644
--- a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt
+++ b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt
@@ -55,17 +55,23 @@ class CrawlerConfig(val filePath: Path) {
.addString("rlpxNetworkInterface", "127.0.0.1", "RLPx network interface", null)
.addListOfString("bootnodes", mainnetEthereumBootnodes, "Bootnodes to discover other peers from", null)
.addString("discoveryDNS", mainnetDiscoveryDNS, "DNS discovery crawler", null)
- .addLong("discoveryDNSPollingPeriod", 60 * 1000, "DNS Discovery Polling Period in milliseconds", null)
+ .addLong("discoveryDNSPollingPeriod", 60 * 1000L, "DNS Discovery Polling Period in milliseconds", null)
.addString(
"jdbcUrl", System.getProperty("DATABASE_URL", System.getenv("DATABASE_URL")),
"JDBC URL of the form jdbc:posgresql://localhost:5432", PropertyValidator.isPresent()
- ).addString("network", "mainnet", "Network to use instead of providing a genesis file.", null)
+ )
+ .addInteger("jdbcConnections", 25, "Number of JDBC connections for the connections pool", null)
+ .addString("network", "mainnet", "Network to use instead of providing a genesis file.", null)
.addString("genesisFile", "", "Genesis file to use in hello", null)
.addInteger("restPort", 1337, "REST port", null)
.addString("restNetworkInterface", "0.0.0.0", "REST network interface", null)
.addString("ethstatsNetworkInterface", "0.0.0.0", "Ethstats network interface", null)
.addInteger("ethstatsPort", 1338, "Ethstats port", null)
.addString("ethstatsSecret", "changeme", "Ethstats shared secret", null)
+ .addLong("peerCacheExpiration", 5 * 60 * 1000L, "Peer data cache expiration", null)
+ .addLong("clientIdsInterval", 24 * 60 * 60 * 1000 * 2L, "Client IDs Interval - number of milliseconds to go back in time", null)
+ .addLong("rlpxDisconnectionDelay", 10 * 1000L, "RLPx connections disconnection delay", null)
+ .addInteger("maxRequestsPerSec", 30, "Number of requests per second over HTTP", null)
.toSchema()
}
@@ -91,4 +97,11 @@ class CrawlerConfig(val filePath: Path) {
fun ethstatsPort() = config.getInteger("ethstatsPort")
fun ethstatsNetworkInterface() = config.getString("ethstatsNetworkInterface")
fun ethstatsSecret() = config.getString("ethstatsSecret")
+
+ fun peerCacheExpiration() = config.getLong("peerCacheExpiration")
+ fun clientIdsInterval() = config.getLong("clientIdsInterval")
+
+ fun jdbcConnections() = config.getInteger("jdbcConnections")
+ fun rlpxDisconnectionDelay() = config.getLong("rlpxDisconnectionsDelay")
+ fun maxRequestsPerSec() = config.getInteger("maxRequestsPerSec")
}
diff --git a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt
index 6704f57..48ba99e 100644
--- a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt
+++ b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt
@@ -38,6 +38,7 @@ class CrawlerRESTService(
val port: Int = 0,
val networkInterface: String = "127.0.0.1",
val path: String = "/",
+ val maxRequestsPerSec: Int = 30,
val repository: RelationalPeerRepository,
override val coroutineContext: CoroutineContext = Dispatchers.Default,
) : CoroutineScope {
@@ -83,8 +84,7 @@ class CrawlerRESTService(
ctx.addServlet(swagger, "/swagger-ui/*")
val filter = DoSFilter()
- // TODO make it a config setting. Change for REST vs UI.
- filter.maxRequestsPerSec = 30
+ filter.maxRequestsPerSec = maxRequestsPerSec
ctx.addFilter(FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST))
newServer.stopAtShutdown = true
diff --git a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
index b2393e8..03c4dc2 100644
--- a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
+++ b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
@@ -43,6 +43,8 @@ import kotlin.coroutines.CoroutineContext
open class RelationalPeerRepository(
private val dataSource: DataSource,
+ private val expiration: Long = 5 * 60 * 1000,
+ private val clientIdsInterval: Long = 24 * 60 * 60 * 1000 * 2,
override val coroutineContext: CoroutineContext = Dispatchers.Default,
) : CoroutineScope, PeerRepository {
@@ -73,7 +75,7 @@ open class RelationalPeerRepository(
}
fun get(nodeId: SECP256K1.PublicKey, endpoint: Endpoint): Peer {
- val id = peerCache.computeIfAbsent(nodeId, 5 * 60 * 60) { // 5 minutes, TODO configure.
+ val id = peerCache.computeIfAbsent(nodeId, expiration) {
dataSource.connection.use { conn ->
logger.trace("Get peer with $nodeId")
val stmt = conn.prepareStatement("select id,publickey from identity where publickey=?")
@@ -261,7 +263,7 @@ open class RelationalPeerRepository(
"select clients.clientId, count(clients.clientId) from (select nodeinfo.clientId, nodeInfo.createdAt from nodeinfo inner join (select identity, max(createdAt) as maxCreatedAt from nodeinfo group by identity) maxSeen on nodeinfo.identity = maxSeen.identity and nodeinfo.createdAt = maxSeen.maxCreatedAt) as clients where clients.createdAt > ? group by clients.clientId"
)
stmt.use {
- it.setTimestamp(1, Timestamp(System.currentTimeMillis() - 24 * 60 * 60 * 1000 * 2)) // TODO configure right now, it's clients from up to 2 days ago.
+ it.setTimestamp(1, Timestamp(System.currentTimeMillis() - clientIdsInterval))
// map results.
val rs = stmt.executeQuery()
val result = mutableListOf<ClientInfo>()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tuweni.apache.org
For additional commands, e-mail: commits-help@tuweni.apache.org