You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by mi...@apache.org on 2021/08/25 21:23:43 UTC
[maven-resolver] 01/01: [MRESOLVER-191] Document how to analyze
lock issues
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch MRESOLVER-191
in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
commit f1c6099a56bba28d236917cbf9af81a2bd0dc27f
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Wed Aug 25 23:19:08 2021 +0200
[MRESOLVER-191] Document how to analyze lock issues
---
.../src/site/markdown/analyzing-lock-issues.md | 108 +++++++++++++++++++++
.../src/site/resources/sql/create_lock_events.sql | 21 ++++
.../site/resources/sql/create_lock_workflows.sql | 19 ++++
.../src/site/resources/sql/schema.sql | 6 ++
maven-resolver-named-locks/src/site/site.xml | 1 +
5 files changed, 155 insertions(+)
diff --git a/maven-resolver-named-locks/src/site/markdown/analyzing-lock-issues.md b/maven-resolver-named-locks/src/site/markdown/analyzing-lock-issues.md
new file mode 100644
index 0000000..83fa43f
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/markdown/analyzing-lock-issues.md
@@ -0,0 +1,108 @@
+# Analyzing Lock Issues
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This guide will show you how to collect and analyze lock issues to report with us.
+
+## Prerequisites
+
+Make sure that the following applies:
+
+- Bash or Bash-compatible installed
+- BSD or GNU userland installed
+- SQLite installed
+- At least Maven 4.0.0-alpha-1 and Maven Resolver 1.7.1 installed
+
+Prepare your environment:
+
+- Set Maven home to `MAVEN_HOME`
+- Select a lock factory, a name mapper, a parallel job count (processes), a build count and set
+ `LOCK_FACTORY`, `NAME_MAPPER`, `JOB_COUNT`, `BUILD_COUNT`
+- Set your root working directory to `ROOT`
+- Set your project name with `PROJECT_NAME`
+- Obtain your project source into `$PROJECT_NAME-1` to `$PROJECT_NAME-$JOB_COUNT`
+- Decide whether each job and build will have its own distinct local repository or a shared one
+ and set it to `LOCAL_REPO`
+- Download [`schema.sql`](./sql/schema.sql), [`create_lock_events.sql`](./sql/create_lock_events.sql),
+ [`create_lock_workflows.sql`](./sql/create_lock_workflows.sql) and set `SCHEMA_FILE`,
+ `CREATE_LOCK_EVENTS_FILE`, `CREATE_LOCK_WORKFLOWS_FILE` to their respective locations
+
+## Running Builds
+
+Open `$JOB_COUNT` terminals and change into the `$ROOT/$PROJECT_NAME-1` to `$ROOT/$PROJECT_NAME-$JOB_COUNT`
+directories. Now start each job in a build loop:
+```
+$ for build in $(eval echo {1..$BUILD_COUNT}); do \
+ # Only if dedicated
+ rm -rf $LOCAL_REPO; \
+ $MAVEN_HOME/bin/mvn clean compile -T1C -B -DskipTests -Dmaven.repo.local=$LOCAL_REPO \
+ -Dorg.slf4j.simpleLogger.showThreadName=true -Dorg.slf4j.simpleLogger.showDateTime=true \
+ -Dorg.slf4j.simpleLogger.log.org.eclipse.aether=trace -l $LOCK_FACTORY-$NAME_MAPPER-$build.log \
+ # Adjust to your needs
+ -Daether.connector.basic.threads=8 -Daether.metadataResolver.threads=8 \
+ # If downloads take longer consider to increase
+ -Daether.syncContext.named.time=120 \
+ -Daether.syncContext.named.factory=$LOCK_FACTORY -Daether.syncContext.named.nameMapper=$NAME_MAPPER; \
+ done
+```
+
+Wait for all jobs to complete.
+
+## Analyzing Builds
+
+We now need to extract, transform and load the data (ETL) into a SQLite database.
+
+Change back to the `$ROOT` directory and collect all log files with build failures:
+```
+$ BUILD_FAILURE_LIST=$(mktemp)
+$ grep -l -r "BUILD FAILURE" $PROJECT_NAME-* > $BUILD_FAILURE_LIST
+```
+Run extraction and transformation:
+```
+$ while read log_file; do \
+ grep -e Acquiring -e Releasing -e "Failed to acquire" $log_file | \
+ sed -e 's# \[#;[#g' -e 's#\] #];#' -e "s#^#$log_file;#" | \
+ sed 's#\[TRACE\];##'; \
+ done < $BUILD_FAILURE_LIST > $LOCK_FACTORY-$NAME_MAPPER-locks.txt
+```
+Load and process the data in SQLite:
+```
+$ SCRIPT=$(mktemp)
+$ cat > $SCRIPT <<EOT
+.read $SCHEMA_FILE
+.separator ;
+.import $LOCK_FACTORY-$NAME_MAPPER-locks.txt lock_history
+.read $CREATE_LOCK_EVENTS_FILE
+.read $CREATE_LOCK_WORKFLOWS_FILE
+.save $LOCK_FACTORY-$NAME_MAPPER-locks.db
+EOT
+$ sqlite3 -batch < $SCRIPT
+$ rm -f $SCRIPT
+```
+
+## Uploading Data
+
+Compress the SQLite database as well as the log files:
+```
+$ tar -caf $LOCK_FACTORY-$NAME_MAPPER-locks.db.xz $LOCK_FACTORY-$NAME_MAPPER-locks.db
+$ tar -caf $LOCK_FACTORY-$NAME_MAPPER-logfiles.tar.xz -T $BUILD_FAILURE_LIST
+$ rm -f $BUILD_FAILURE_LIST
+```
+and upload them to the JIRA issue.
diff --git a/maven-resolver-named-locks/src/site/resources/sql/create_lock_events.sql b/maven-resolver-named-locks/src/site/resources/sql/create_lock_events.sql
new file mode 100644
index 0000000..425851e
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/resources/sql/create_lock_events.sql
@@ -0,0 +1,21 @@
+create table lock_events as
+select file, timestamp, substr(thread, 2, length(thread)-2) as thread,
+(case
+when message like 'Acquiring%'
+ then 'acquire'
+when message like 'Releasing%'
+ then 'release'
+when message like 'Failed to acquire%'
+ then 'acquire_failed'
+end) as lock_event,
+(case
+when message like '%write%'
+ then 'write'
+when message like '%read%'
+ then 'read'
+end) as lock_mode,
+(substr(
+ substr(message, instr(message, '''') + 1), 1,
+ instr(substr(message, instr(message, '''') + 1), '''') - 1)
+) as lock_name
+from lock_history;
diff --git a/maven-resolver-named-locks/src/site/resources/sql/create_lock_workflows.sql b/maven-resolver-named-locks/src/site/resources/sql/create_lock_workflows.sql
new file mode 100644
index 0000000..a6cb563
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/resources/sql/create_lock_workflows.sql
@@ -0,0 +1,19 @@
+create table lock_workflows as
+select lock_held, file, timestamp as start_timestamp, "timestamp:1" as end_timestamp, thread, (lock_event || ' => ' || "lock_event:1") as lock_workflow, lock_mode, lock_name
+from (
+ select
+ row_number() over (
+ partition by a.file, a.timestamp, a.thread, a.lock_event, a.lock_mode, a.lock_name
+ order by r.timestamp asc
+ ) as rn,
+ (r.timestamp - a.timestamp) as lock_held, a.*, r.*
+ from lock_events a, lock_events r
+ where r.file = a.file
+ and r.thread = a.thread
+ and r.timestamp >= a.timestamp
+ and a.lock_event = 'acquire' and r.lock_event in ('release', 'acquire_failed')
+ and a.lock_mode = r.lock_mode
+ and a.lock_name = r.lock_name
+)
+where rn = 1
+order by timestamp;
diff --git a/maven-resolver-named-locks/src/site/resources/sql/schema.sql b/maven-resolver-named-locks/src/site/resources/sql/schema.sql
new file mode 100644
index 0000000..456b04b
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/resources/sql/schema.sql
@@ -0,0 +1,6 @@
+create table "lock_history" (
+ "file" text not null,
+ "timestamp" integer not null,
+ "thread" text not null,
+ "message" text not null
+)
diff --git a/maven-resolver-named-locks/src/site/site.xml b/maven-resolver-named-locks/src/site/site.xml
index efed113..9dfc2e8 100644
--- a/maven-resolver-named-locks/src/site/site.xml
+++ b/maven-resolver-named-locks/src/site/site.xml
@@ -26,6 +26,7 @@ under the License.
<body>
<menu name="Overview">
<item name="Introduction" href="index.html"/>
+ <item name="Analyzing Lock Issues" href="analyzing-lock-issues.html"/>
<item name="Javadoc" href="apidocs/index.html"/>
<item name="Source Xref" href="xref/index.html"/>
<!--item name="FAQ" href="faq.html"/-->