You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by mi...@apache.org on 2021/08/25 21:23:42 UTC
[maven-resolver] branch MRESOLVER-191 created (now f1c6099)
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a change to branch MRESOLVER-191
in repository https://gitbox.apache.org/repos/asf/maven-resolver.git.
at f1c6099 [MRESOLVER-191] Document how to analyze lock issues
This branch includes the following new commits:
new f1c6099 [MRESOLVER-191] Document how to analyze lock issues
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[maven-resolver] 01/01: [MRESOLVER-191] Document how to analyze
lock issues
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelo pushed a commit to branch MRESOLVER-191
in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
commit f1c6099a56bba28d236917cbf9af81a2bd0dc27f
Author: Michael Osipov <mi...@apache.org>
AuthorDate: Wed Aug 25 23:19:08 2021 +0200
[MRESOLVER-191] Document how to analyze lock issues
---
.../src/site/markdown/analyzing-lock-issues.md | 108 +++++++++++++++++++++
.../src/site/resources/sql/create_lock_events.sql | 21 ++++
.../site/resources/sql/create_lock_workflows.sql | 19 ++++
.../src/site/resources/sql/schema.sql | 6 ++
maven-resolver-named-locks/src/site/site.xml | 1 +
5 files changed, 155 insertions(+)
diff --git a/maven-resolver-named-locks/src/site/markdown/analyzing-lock-issues.md b/maven-resolver-named-locks/src/site/markdown/analyzing-lock-issues.md
new file mode 100644
index 0000000..83fa43f
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/markdown/analyzing-lock-issues.md
@@ -0,0 +1,108 @@
+# Analyzing Lock Issues
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This guide will show you how to collect and analyze lock issues to report with us.
+
+## Prerequisites
+
+Make sure that the following applies:
+
+- Bash or Bash-compatible installed
+- BSD or GNU userland installed
+- SQLite installed
+- At least Maven 4.0.0-alpha-1 and Maven Resolver 1.7.1 installed
+
+Prepare your environment:
+
+- Set Maven home to `MAVEN_HOME`
+- Select a lock factory, a name mapper, a parallel job count (processes), a build count and set
+ `LOCK_FACTORY`, `NAME_MAPPER`, `JOB_COUNT`, `BUILD_COUNT`
+- Set your root working directory to `ROOT`
+- Set your project name with `PROJECT_NAME`
+- Obtain your project source into `$PROJECT_NAME-1` to `$PROJECT_NAME-$JOB_COUNT`
+- Decide whether each job and build will have its own distinct local repository or a shared one
+ and set it to `LOCAL_REPO`
+- Download [`schema.sql`](./sql/schema.sql), [`create_lock_events.sql`](./sql/create_lock_events.sql),
+ [`create_lock_workflows.sql`](./sql/create_lock_workflows.sql) and set `SCHEMA_FILE`,
+ `CREATE_LOCK_EVENTS_FILE`, `CREATE_LOCK_WORKFLOWS_FILE` to their respective locations
+
+## Running Builds
+
+Open `$JOB_COUNT` terminals and change into the `$ROOT/$PROJECT_NAME-1` to `$ROOT/$PROJECT_NAME-$JOB_COUNT`
+directories. Now start each job in a build loop:
+```
+$ for build in $(eval echo {1..$BUILD_COUNT}); do \
+ # Only if dedicated
+ rm -rf $LOCAL_REPO; \
+ $MAVEN_HOME/bin/mvn clean compile -T1C -B -DskipTests -Dmaven.repo.local=$LOCAL_REPO \
+ -Dorg.slf4j.simpleLogger.showThreadName=true -Dorg.slf4j.simpleLogger.showDateTime=true \
+ -Dorg.slf4j.simpleLogger.log.org.eclipse.aether=trace -l $LOCK_FACTORY-$NAME_MAPPER-$build.log \
+ # Adjust to your needs
+ -Daether.connector.basic.threads=8 -Daether.metadataResolver.threads=8 \
+ # If downloads take longer consider to increase
+ -Daether.syncContext.named.time=120 \
+ -Daether.syncContext.named.factory=$LOCK_FACTORY -Daether.syncContext.named.nameMapper=$NAME_MAPPER; \
+ done
+```
+
+Wait for all jobs to complete.
+
+## Analyzing Builds
+
+We now need to extract, transform and load the data (ETL) into a SQLite database.
+
+Change back to the `$ROOT` directory and collect all log files with build failures:
+```
+$ BUILD_FAILURE_LIST=$(mktemp)
+$ grep -l -r "BUILD FAILURE" $PROJECT_NAME-* > $BUILD_FAILURE_LIST
+```
+Run extraction and transformation:
+```
+$ while read log_file; do \
+ grep -e Acquiring -e Releasing -e "Failed to acquire" $log_file | \
+ sed -e 's# \[#;[#g' -e 's#\] #];#' -e "s#^#$log_file;#" | \
+ sed 's#\[TRACE\];##'; \
+ done < $BUILD_FAILURE_LIST > $LOCK_FACTORY-$NAME_MAPPER-locks.txt
+```
+Load and process the data in SQLite:
+```
+$ SCRIPT=$(mktemp)
+$ cat > $SCRIPT <<EOT
+.read $SCHEMA_FILE
+.separator ;
+.import $LOCK_FACTORY-$NAME_MAPPER-locks.txt lock_history
+.read $CREATE_LOCK_EVENTS_FILE
+.read $CREATE_LOCK_WORKFLOWS_FILE
+.save $LOCK_FACTORY-$NAME_MAPPER-locks.db
+EOT
+$ sqlite3 -batch < $SCRIPT
+$ rm -f $SCRIPT
+```
+
+## Uploading Data
+
+Compress the SQLite database as well as the log files:
+```
+$ tar -caf $LOCK_FACTORY-$NAME_MAPPER-locks.db.xz $LOCK_FACTORY-$NAME_MAPPER-locks.db
+$ tar -caf $LOCK_FACTORY-$NAME_MAPPER-logfiles.tar.xz -T $BUILD_FAILURE_LIST
+$ rm -f $BUILD_FAILURE_LIST
+```
+and upload them to the JIRA issue.
diff --git a/maven-resolver-named-locks/src/site/resources/sql/create_lock_events.sql b/maven-resolver-named-locks/src/site/resources/sql/create_lock_events.sql
new file mode 100644
index 0000000..425851e
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/resources/sql/create_lock_events.sql
@@ -0,0 +1,21 @@
+create table lock_events as
+select file, timestamp, substr(thread, 2, length(thread)-2) as thread,
+(case
+when message like 'Acquiring%'
+ then 'acquire'
+when message like 'Releasing%'
+ then 'release'
+when message like 'Failed to acquire%'
+ then 'acquire_failed'
+end) as lock_event,
+(case
+when message like '%write%'
+ then 'write'
+when message like '%read%'
+ then 'read'
+end) as lock_mode,
+(substr(
+ substr(message, instr(message, '''') + 1), 1,
+ instr(substr(message, instr(message, '''') + 1), '''') - 1)
+) as lock_name
+from lock_history;
diff --git a/maven-resolver-named-locks/src/site/resources/sql/create_lock_workflows.sql b/maven-resolver-named-locks/src/site/resources/sql/create_lock_workflows.sql
new file mode 100644
index 0000000..a6cb563
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/resources/sql/create_lock_workflows.sql
@@ -0,0 +1,19 @@
+create table lock_workflows as
+select lock_held, file, timestamp as start_timestamp, "timestamp:1" as end_timestamp, thread, (lock_event || ' => ' || "lock_event:1") as lock_workflow, lock_mode, lock_name
+from (
+ select
+ row_number() over (
+ partition by a.file, a.timestamp, a.thread, a.lock_event, a.lock_mode, a.lock_name
+ order by r.timestamp asc
+ ) as rn,
+ (r.timestamp - a.timestamp) as lock_held, a.*, r.*
+ from lock_events a, lock_events r
+ where r.file = a.file
+ and r.thread = a.thread
+ and r.timestamp >= a.timestamp
+ and a.lock_event = 'acquire' and r.lock_event in ('release', 'acquire_failed')
+ and a.lock_mode = r.lock_mode
+ and a.lock_name = r.lock_name
+)
+where rn = 1
+order by timestamp;
diff --git a/maven-resolver-named-locks/src/site/resources/sql/schema.sql b/maven-resolver-named-locks/src/site/resources/sql/schema.sql
new file mode 100644
index 0000000..456b04b
--- /dev/null
+++ b/maven-resolver-named-locks/src/site/resources/sql/schema.sql
@@ -0,0 +1,6 @@
+create table "lock_history" (
+ "file" text not null,
+ "timestamp" integer not null,
+ "thread" text not null,
+ "message" text not null
+)
diff --git a/maven-resolver-named-locks/src/site/site.xml b/maven-resolver-named-locks/src/site/site.xml
index efed113..9dfc2e8 100644
--- a/maven-resolver-named-locks/src/site/site.xml
+++ b/maven-resolver-named-locks/src/site/site.xml
@@ -26,6 +26,7 @@ under the License.
<body>
<menu name="Overview">
<item name="Introduction" href="index.html"/>
+ <item name="Analyzing Lock Issues" href="analyzing-lock-issues.html"/>
<item name="Javadoc" href="apidocs/index.html"/>
<item name="Source Xref" href="xref/index.html"/>
<!--item name="FAQ" href="faq.html"/-->