You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nuttx.apache.org by GitBox <gi...@apache.org> on 2020/10/07 07:41:59 UTC

[GitHub] [incubator-nuttx-apps] YuuichiNakamura opened a new pull request #421: Add task trace support

YuuichiNakamura opened a new pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421


   ## Summary
   This PR adds "trace" command for task trace.
   
   - Currently the documentation is available at https://github.com/YuuichiNakamura/nuttx-task-tracer-doc/blob/master/NuttXTaskTracer.md 
   - rst version will be prepared later.
   
   ## Impact
   - "trace" command is added to nsh. It is available only when the related kernel configurations are enabled.
   
   ## Testing
   Tested by spresense:nsh by changing the following configurations.
   ```
   CONFIG_SCHED_INSTRUMENTATION=y
   CONFIG_SCHED_INSTRUMENTATION_FILTER=y
   CONFIG_SCHED_INSTRUMENTATION_SYSCALL=y
   CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER=y
   CONFIG_DRIVER_NOTE=y
   CONFIG_DRIVER_NOTERAM=y
   CONFIG_DRIVER_NOTECTL=y
   ```
   `trace` command is added to nsh by this configuration.
   ```
   nsh> trace cmd "usleep 1000"
   nsh> trace dump
   ```
   This can get the following results:
   ```
   <noname>-1   [0]   7.640000000: sys_ioctl -> 0
   <noname>-1   [0]   7.640000000: sys_close()
   <noname>-1   [0]   7.640000000: sys_close -> 0
   <noname>-1   [0]   7.640000000: sys_sched_lock()
   <noname>-1   [0]   7.640000000: sys_sched_lock -> 0
   <noname>-1   [0]   7.640000000: sys_nxsched_get_stackinfo()
   <noname>-1   [0]   7.640000000: sys_nxsched_get_stackinfo -> 0
   <noname>-1   [0]   7.640000000: sys_sched_unlock()
   <noname>-1   [0]   7.640000000: sys_sched_unlock -> 0
   <noname>-1   [0]   7.640000000: sys_clock_nanosleep()
   <noname>-1   [0]   7.640000000: sched_switch: prev_comm=<noname> prev_pid=1 prev_state=S ==> next_comm=<noname> next_pid=0
   <noname>-0   [0]   7.640000000: irq_handler_entry: irq=11
   <noname>-0   [0]   7.640000000: irq_handler_exit: irq=11
   <noname>-0   [0]   7.640000000: irq_handler_entry: irq=15
   <noname>-0   [0]   7.650000000: irq_handler_exit: irq=15
   <noname>-0   [0]   7.650000000: irq_handler_entry: irq=15
   <noname>-0   [0]   7.660000000: sched_waking: comm=<noname> pid=1 target_cpu=0
   <noname>-1   [0]   7.660000000: irq_handler_exit: irq=15
   <noname>-1   [0]   7.660000000: sched_switch: prev_comm=<noname> prev_pid=1 prev_state=R ==> next_comm=<noname> next_pid=1
   <noname>-1   [0]   7.660000000: sys_clock_nanosleep -> 0
   ```
   
   


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r505123430



##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p,
+                          FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      cctx->current_pid = pid;

Review comment:
       It is needed because cctx->current_pid isn't updated until the first context switch in the trace.
   The trace data begins in incomplete state and the beginning of the trace dump also shows incomplete result. This line fixes such situation though it is not perfect.
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r501524860



##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       Sure, but your tool is a superset of sched_note, should we merge both into one to reduce the maintainence cost? BTW, sched_node may already in broken state with the recent note change.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r505144751



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>

Review comment:
       How about we change trace_dump_get_mode to trace_dump_get_overwrite or trace_dump_is_overwrite? so true/false is used in trace.c and trace_dump.c convert true/false to NOTERAM_MODE_OVERWRITE?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503733998



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);

Review comment:
       It is to avoid to record the ioctl which controls a trace mode.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r505121803



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);

Review comment:
       That's true... I didn't like remaining an irrelevant ioctls in log but consistency is important.
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503735944



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"

Review comment:
       I tried to change trace_cmd_dump() to move overwrite option from trace_cmd_mode().
   But, I think it should not be moved into trace_dump.c. Because,
   - The command line parsing is split into two files.
   - trace_cmd_dump() needs to call notectl_enable() because https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503734494 . Then, both files need to call another mutually and it becomes not simple.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503735944



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"

Review comment:
       I tried to change trace_cmd_dump() to move overwrite option from trace_cmd_mode().
   But, I think it should not be moved into trace_dump.c. Because,
   - The command line parsing is split into two files.
   - trace_cmd_dump() needs to call notectl_enable() because https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r502805629 . Then, both files need to call another mutually and it becomes not simple.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r505121937



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>

Review comment:
       It is still needed because NOTERAM_MODE_OVERWRITE_ENABLE/DISABLE definition is used in trace_cmd_mode().
   
   

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static int notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  trace_dump_clear();

Review comment:
       Adding "-c" (continue trace) option seems good.

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static int notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  trace_dump_clear();
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true, notectlfd);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  FAR FILE *out = stdout;
+  unsigned int owmode;
+  bool enable;
+  int ret;
+
+  /* Usage: trace dump [{+|-}o] [<filename>] */
+
+  if (index == argc - 1)
+    {
+      /* No argument - display current overwrite mode */
+
+      owmode = trace_dump_getmode();
+      printf("Trace dump mode:\n");
+      printf(" Overwrite               : %s\n",
+             (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+              "on  (+o)" : "off (-o)");
+
+      return index;
+    }
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace dump: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      trace_dump_setmode(owmode);
+    }
+
+  if (index == argc - 1)
+    {
+      /* <filename> is not given - change overwrite mode only */
+
+      return index;
+    }
+
+  /* If <filename> is given, open the file stream for output */
+
+  index++;
+  out = fopen(argv[index], "w");
+  if (out == NULL)
+    {
+      fprintf(stderr,
+              "trace dump: cannot open '%s'\n", argv[index]);
+      return ERROR;
+    }
+
+  /* Stop the tracing before dump */

Review comment:
       I'd like to add a similar option like "trace start -c".




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503734423



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)

Review comment:
       I got it. But line 35-37 cannot be removed because NOTERAM_MODE_OVERWRITE defines are still required.
   
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503733788



##########
File path: system/trace/trace.h
##########
@@ -0,0 +1,59 @@
+/****************************************************************************
+ * apps/system/trace/trace.h
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+#ifndef __APPS_SYSTEM_TRACE_TRACE_H
+#define __APPS_SYSTEM_TRACE_TRACE_H
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+
+#ifdef __cplusplus
+#define EXTERN extern "C"
+extern "C"
+{
+#else
+#define EXTERN extern
+#endif
+
+/****************************************************************************
+ * Public Function Prototypes
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: trace_dump
+ *
+ * Description:
+ *   Read notes and dump trace results.
+ *
+ ****************************************************************************/
+
+int trace_dump(FILE *out);

Review comment:
       If the whole file is guarded in Makefile (https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r502535529), I think we have no need to guard here.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#issuecomment-705383757


   @patacongo I have fixed the implementation as the built-in apps and moved into apps/system/trace.
   @xiaoxiang781216 Some of your comments were solved by changing into built-in apps.
   


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r502208254



##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       sched_note shows the scheduler notes as it is and I think it is important for kernel development, especially for scheduler.
   Different to it, my command processes and replaces the notes to fit for tracecompass. Moreover, basically my command intends to handle the notes after the trace is stopped. This is a different behavior to sched_note. So I'm negative to merge them into one.
   sched_note is still working. My changes for note device doesn't affect the existing behavior.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#issuecomment-705383757






----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503735462



##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,570 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <nuttx/sched_note.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+static FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;

Review comment:
       You mean both g_dump_cpu_ctx[] and g_dump_task_ctx?
   It seems a bit complicated to add both argument to every private functions.
   I think we have no strong reason to change into local variable because:
   - These are widely accessed by private functions in this file, 
   - Already defined as static and used only in this file.
   - The function never called recursively.
   
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r504485072



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"

Review comment:
       No problem.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r505152637



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,775 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static bool notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+  bool cont = false;
+
+  /* Usage: trace start [-c][<duration>] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  if (index < argc)
+    {
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM

Review comment:
       how about we change trace.h like this:
   ```
   #ifdef CONFIG_DRIVER_NOTERAM
   void trace_dump_clear(void);
   #else
   #define trace_dump_clear
   #endif
   ```
   so we don't need check CONFIG_DRIVER_NOTERAM again and again. other functions in trace.h could apply the approach.

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,775 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static bool notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+  bool cont = false;
+
+  /* Usage: trace start [-c][<duration>] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  if (index < argc)
+    {
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (!cont)
+    {
+      trace_dump_clear();
+    }
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true, notectlfd);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  FAR FILE *out = stdout;
+  int ret;
+  bool changed;
+  bool cont = false;
+
+  /* Usage: trace dump [-c][<filename>] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+
+      index++;
+    }
+
+  /* Stop the tracing before dump */
+
+  if (!cont)
+    {
+      changed = notectl_enable(false, notectlfd);
+    }
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  if (!cont && changed)

Review comment:
       let's change line 152 to: bool changed = false and remove !cont &&?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,775 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static bool notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+  bool cont = false;
+
+  /* Usage: trace start [-c][<duration>] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  if (index < argc)
+    {
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (!cont)
+    {
+      trace_dump_clear();
+    }
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true, notectlfd);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  FAR FILE *out = stdout;
+  int ret;
+  bool changed;
+  bool cont = false;
+
+  /* Usage: trace dump [-c][<filename>] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+
+      index++;
+    }
+
+  /* Stop the tracing before dump */
+
+  if (!cont)
+    {
+      changed = notectl_enable(false, notectlfd);
+    }
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  if (!cont && changed)
+    {
+      notectl_enable(true, notectlfd);
+    }
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, FAR char **argv, int notectlfd)
+{
+  char command[CONFIG_NSH_LINELEN];
+  bool changed;
+  bool cont = false;
+
+  /* Usage: trace cmd [-c] <command> [<args>...] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (!cont)
+    {
+      trace_dump_clear();
+    }
+#endif
+
+  /* Execute the command with tracing */
+
+  changed = notectl_enable(true, notectlfd);
+
+  system(command);
+
+  if (changed)
+    {
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  owmode = trace_dump_getmode();

Review comment:
       same comment as line 120

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,775 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static bool notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+  bool cont = false;
+
+  /* Usage: trace start [-c][<duration>] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  if (index < argc)
+    {
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (!cont)
+    {
+      trace_dump_clear();
+    }
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true, notectlfd);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  FAR FILE *out = stdout;
+  int ret;
+  bool changed;
+  bool cont = false;
+
+  /* Usage: trace dump [-c][<filename>] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+
+      index++;
+    }
+
+  /* Stop the tracing before dump */
+
+  if (!cont)
+    {
+      changed = notectl_enable(false, notectlfd);
+    }
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  if (!cont && changed)
+    {
+      notectl_enable(true, notectlfd);
+    }
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, FAR char **argv, int notectlfd)
+{
+  char command[CONFIG_NSH_LINELEN];
+  bool changed;
+  bool cont = false;
+
+  /* Usage: trace cmd [-c] <command> [<args>...] */
+
+  if (index < argc)
+    {
+      if (strcmp(argv[index], "-c") == 0)
+        {
+          cont = true;
+          index++;
+        }
+    }
+
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (!cont)
+    {
+      trace_dump_clear();

Review comment:
       same comment as line 120




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r500839342



##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,
+    "[start [<duration>]] [stop] "
+#  ifdef CONFIG_DRIVER_NOTE

Review comment:
       CONFIG_DRIVER_NOTE to CONFIG_DRIVER_NOTERAM?

##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       should we rename all trace to node which is better align with the kernel term?

##########
File path: nshlib/nsh_trace.c
##########
@@ -0,0 +1,1326 @@
+/****************************************************************************
+ * apps/nshlib/nsh_trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <time.h>
+#include <nuttx/clock.h>
+
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+#ifdef CONFIG_DRIVER_NOTECTL
+#  include <nuttx/note/notectl_driver.h>
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+#include "nsh.h"
+#include "nsh_console.h"
+
+#ifdef CONFIG_DRIVER_NOTECTL
+#  ifndef CONFIG_NSH_DISABLE_TRACE
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+#endif
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+#endif
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;
+#endif
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd >= 0)
+    {
+      ioctl(fd, cmd, arg);
+      close(fd);
+    }
+}
+#endif
+
+#ifdef CONFIG_DRIVER_NOTERAM
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(void)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      g_dump_cpu_ctx[cpu].ininterrupt = false;
+      g_dump_cpu_ctx[cpu].pendingswitch = false;
+      g_dump_cpu_ctx[cpu].current_state = TSTATE_TASK_RUNNING;
+      g_dump_cpu_ctx[cpu].current_pid = 0;
+      g_dump_cpu_ctx[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(void)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = g_dump_task_ctx;
+  g_dump_task_ctx = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &g_dump_task_ctx;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s *tctxp;
+
+  tctxp = get_task_context(pid);
+  if (tctxp != NULL && tctxp->name[0] != '\0')
+    {
+      return tctxp->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = g_dump_cpu_ctx[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note)
+{
+  FAR struct trace_dump_cpu_context_s *ctxp;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  ctxp = &g_dump_cpu_ctx[cpu];
+  current_pid = ctxp->current_pid;
+  next_pid = ctxp->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid), get_pid(current_pid),
+          get_task_state(ctxp->current_state),
+          get_task_name(next_pid), get_pid(next_pid));
+
+  ctxp->current_pid = ctxp->next_pid;
+  ctxp->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *ctxp;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  ctxp = &g_dump_cpu_ctx[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      ctxp->current_pid = pid;
+    }
+
+  /* Output one note */
+
+  switch (note->nc_type)
+    {
+      case NOTE_START:
+        {
+#if CONFIG_TASK_NAME_SIZE > 0
+          FAR struct note_start_s *nst = (FAR struct note_start_s *)p;
+          FAR struct trace_dump_task_context_s *tctxp;
+
+          tctxp = get_task_context(pid);
+          if (tctxp != NULL)
+            {
+              copy_task_name(tctxp->name, nst->nst_name);
+            }
+#endif
+
+          trace_dump_header(out, note);
+          fprintf(out, "sched_wakeup_new: comm=%s pid=%d target_cpu=%d\n",
+                  get_task_name(pid), get_pid(pid), cpu);
+        }
+        break;
+
+      case NOTE_STOP:
+        {
+          trace_dump_header(out, note);
+          fprintf(out, "sched_switch: "
+                       "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+                       "next_comm=%s next_pid=%u\n",
+                  get_task_name(pid), get_pid(pid), 'X',
+                  get_task_name(ctxp->current_pid),
+                  get_pid(ctxp->current_pid));
+        }
+        break;
+
+      case NOTE_SUSPEND:
+        {
+          FAR struct note_suspend_s *nsu = (FAR struct note_suspend_s *)p;
+
+          /* This note informs the task to be suspended.
+           * Preserve the information for the succeeding NOTE_RESUME.
+           */
+
+          ctxp->current_state = nsu->nsu_state;
+        }
+        break;
+
+      case NOTE_RESUME:
+        {
+          /* This note informs the task to be resumed.
+           * The task switch timing depends on the running context.
+           */
+
+          ctxp->next_pid = pid;
+
+          if (!ctxp->ininterrupt)
+            {
+              /* If not in the interrupt context, the task switch is
+               * executed immediately.
+               */
+
+              trace_dump_header(out, note);
+              trace_dump_sched_switch(out, note);
+            }
+          else
+            {
+              /* If in the interrupt context, the task switch is postponed
+               * until leaving the interrupt handler.
+               */
+
+              trace_dump_header(out, note);
+              fprintf(out, "sched_waking: comm=%s pid=%d target_cpu=%d\n",
+                      get_task_name(ctxp->next_pid),
+                      get_pid(ctxp->next_pid), cpu);
+              ctxp->pendingswitch = true;
+            }
+        }
+        break;
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      case NOTE_SYSCALL_ENTER:
+        {
+          FAR struct note_syscall_enter_s *nsc;
+          FAR struct trace_dump_task_context_s *tctxp;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (ctxp->ininterrupt)
+            {
+              break;
+            }
+
+          tctxp = get_task_context(pid);
+          if (tctxp == NULL)
+            {
+              break;
+            }
+
+          tctxp->syscall_nest++;
+          if (tctxp->syscall_nest > 1)
+            {
+              break;
+            }
+
+          nsc = (FAR struct note_syscall_enter_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "sys_%s()\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED]);
+        }
+        break;
+
+      case NOTE_SYSCALL_LEAVE:
+        {
+          FAR struct note_syscall_leave_s *nsc;
+          FAR struct trace_dump_task_context_s *tctxp;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (ctxp->ininterrupt)
+            {
+              break;
+            }
+
+          tctxp = get_task_context(pid);
+          if (tctxp == NULL)
+            {
+              break;
+            }
+
+          tctxp->syscall_nest--;
+          if (tctxp->syscall_nest > 0)
+            {
+              break;
+            }
+
+          tctxp->syscall_nest = 0;
+
+          nsc = (FAR struct note_syscall_leave_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "sys_%s -> 0x%x\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED],
+                  nsc->nsc_result);
+        }
+        break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+      case NOTE_IRQ_ENTER:
+        {
+          FAR struct note_irqhandler_s *nih;
+
+          nih = (FAR struct note_irqhandler_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "irq_handler_entry: irq=%u\n",
+                  nih->nih_irq);
+          ctxp->ininterrupt = true;
+        }
+        break;
+
+      case NOTE_IRQ_LEAVE:
+        {
+          FAR struct note_irqhandler_s *nih;
+
+          nih = (FAR struct note_irqhandler_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "irq_handler_exit: irq=%u\n",
+                  nih->nih_irq);
+          ctxp->ininterrupt = false;
+          if (ctxp->pendingswitch)
+            {
+              /* If the pending task switch exists, it is executed here */
+
+              trace_dump_header(out, note);
+              trace_dump_sched_switch(out, note);
+            }
+        }
+        break;
+#endif
+
+      default:
+        break;
+    }
+
+  /* Return the length of the processed note */
+
+  return note->nc_length;
+}
+
+/****************************************************************************
+ * Name: trace_dump
+ ****************************************************************************/
+
+static int trace_dump(FILE *out)
+{
+  uint8_t tracedata[UCHAR_MAX];
+  FAR uint8_t *p;
+  int size;
+  int ret;
+  int fd;
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Open note for read */
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      return ERROR;
+    }
+
+  trace_dump_init_context();
+
+  /* Read and output all notes */
+
+  while (1)
+    {
+      ret = read(fd, tracedata, sizeof tracedata);
+      if (ret <= 0)
+        {
+          break;
+        }
+
+      p = tracedata;
+      do
+        {
+          size = trace_dump_one(out, p);
+          p += size;
+          ret -= size;
+        }
+      while (ret > 0);
+    }
+
+  trace_dump_fini_context();
+
+  /* Close note */
+
+  close(fd);
+
+  return ret;
+}
+
+#endif /* CONFIG_DRIVER_NOTERAM */
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(FAR struct nsh_vtbl_s *vtbl,
+                           int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          nsh_output(vtbl, g_fmtarginvalid, argv[0]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(FAR struct nsh_vtbl_s *vtbl,
+                          int index, int argc, char **argv)
+{
+  FAR char *fullpath;
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          fullpath = nsh_getfullpath(vtbl, argv[index]);
+          if (fullpath == NULL)
+            {
+              nsh_output(vtbl, g_fmtcmdoutofmemory, argv[0]);
+              return ERROR;
+            }
+
+          out = fopen(fullpath, "w");
+          if (out == NULL)
+            {
+              nsh_output(vtbl, g_fmtcmdfailed, argv[0], "open", NSH_ERRNO);
+              return ERROR;
+            }
+
+          nsh_freefullpath(fullpath);
+        }
+    }
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      nsh_output(vtbl, g_fmtcmdfailed, argv[0], "open", NSH_ERRNO);
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+static int trace_cmd_cmd(FAR struct nsh_vtbl_s *vtbl,
+                         int index, int argc, char **argv)
+{
+#ifndef CONFIG_NSH_DISABLEBG
+  bool bgsave;
+#endif
+#if CONFIG_FILE_STREAM
+  bool redirsave;
+#endif
+  int ret;
+
+  /* Usage: trace cmd "<command>" */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      nsh_output(vtbl, g_fmtargrequired, argv[0]);
+      return ERROR;
+    }
+
+  /* Save state */
+
+#ifndef CONFIG_NSH_DISABLEBG
+  bgsave    = vtbl->np.np_bg;
+#endif
+#if CONFIG_FILE_STREAM
+  redirsave = vtbl->np.np_redirect;
+#endif
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  ret = nsh_parse(vtbl, argv[index]);
+  notectl_enable(false);
+
+  /* Restore state */
+
+#ifndef CONFIG_NSH_DISABLEBG
+  vtbl->np.np_bg       = bgsave;
+#endif
+#if CONFIG_FILE_STREAM
+  vtbl->np.np_redirect = redirsave;
+#endif
+
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(FAR struct nsh_vtbl_s *vtbl,
+                          int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            nsh_output(vtbl, g_fmtsyntax, argv[0]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  nsh_output(vtbl, "Task trace mode:\n");
+  nsh_output(vtbl, " Trace                   : %s\n",
+             (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+              "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  nsh_output(vtbl, " Overwrite               : %s\n",
+             (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+              "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  nsh_output(vtbl, " Syscall trace           : %s\n",
+             mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+               "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      nsh_output(vtbl, "  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  nsh_output(vtbl, " IRQ trace               : %s\n",
+             mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+               "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      nsh_output(vtbl, "  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(FAR struct nsh_vtbl_s *vtbl,
+                            int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      nsh_output(vtbl, "Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              nsh_output(vtbl, "  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(FAR struct nsh_vtbl_s *vtbl,
+                         int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          nsh_output(vtbl, g_fmtarginvalid, argv[0]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      nsh_output(vtbl, "Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              nsh_output(vtbl, "  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: cmd_trace
+ ****************************************************************************/
+
+int cmd_trace(FAR struct nsh_vtbl_s *vtbl, int argc, char **argv)
+{
+  int ret = OK;
+  int i;
+
+  /* Open note control device */
+
+  g_notectlfd = open("/dev/notectl", 0);

Review comment:
       change g_notectlfd  to local variable and pass it to other function as argument?

##########
File path: nshlib/nsh_trace.c
##########
@@ -0,0 +1,1326 @@
+/****************************************************************************
+ * apps/nshlib/nsh_trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <time.h>
+#include <nuttx/clock.h>
+
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+#ifdef CONFIG_DRIVER_NOTECTL
+#  include <nuttx/note/notectl_driver.h>
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+#include "nsh.h"
+#include "nsh_console.h"
+
+#ifdef CONFIG_DRIVER_NOTECTL

Review comment:
       don't need? guard by Makefile already.

##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       Sure, but your tool is a superset of sched_note, should we merge both into one to reduce the maintainence cost? BTW, sched_node may already in broken state with the recent note change.

##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       Ok.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r500839342



##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,
+    "[start [<duration>]] [stop] "
+#  ifdef CONFIG_DRIVER_NOTE

Review comment:
       CONFIG_DRIVER_NOTE to CONFIG_DRIVER_NOTERAM?

##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       should we rename all trace to node which is better align with the kernel term?

##########
File path: nshlib/nsh_trace.c
##########
@@ -0,0 +1,1326 @@
+/****************************************************************************
+ * apps/nshlib/nsh_trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <time.h>
+#include <nuttx/clock.h>
+
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+#ifdef CONFIG_DRIVER_NOTECTL
+#  include <nuttx/note/notectl_driver.h>
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+#include "nsh.h"
+#include "nsh_console.h"
+
+#ifdef CONFIG_DRIVER_NOTECTL
+#  ifndef CONFIG_NSH_DISABLE_TRACE
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+#endif
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+#endif
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;
+#endif
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd >= 0)
+    {
+      ioctl(fd, cmd, arg);
+      close(fd);
+    }
+}
+#endif
+
+#ifdef CONFIG_DRIVER_NOTERAM
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(void)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      g_dump_cpu_ctx[cpu].ininterrupt = false;
+      g_dump_cpu_ctx[cpu].pendingswitch = false;
+      g_dump_cpu_ctx[cpu].current_state = TSTATE_TASK_RUNNING;
+      g_dump_cpu_ctx[cpu].current_pid = 0;
+      g_dump_cpu_ctx[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(void)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = g_dump_task_ctx;
+  g_dump_task_ctx = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &g_dump_task_ctx;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s *tctxp;
+
+  tctxp = get_task_context(pid);
+  if (tctxp != NULL && tctxp->name[0] != '\0')
+    {
+      return tctxp->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = g_dump_cpu_ctx[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note)
+{
+  FAR struct trace_dump_cpu_context_s *ctxp;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  ctxp = &g_dump_cpu_ctx[cpu];
+  current_pid = ctxp->current_pid;
+  next_pid = ctxp->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid), get_pid(current_pid),
+          get_task_state(ctxp->current_state),
+          get_task_name(next_pid), get_pid(next_pid));
+
+  ctxp->current_pid = ctxp->next_pid;
+  ctxp->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *ctxp;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  ctxp = &g_dump_cpu_ctx[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      ctxp->current_pid = pid;
+    }
+
+  /* Output one note */
+
+  switch (note->nc_type)
+    {
+      case NOTE_START:
+        {
+#if CONFIG_TASK_NAME_SIZE > 0
+          FAR struct note_start_s *nst = (FAR struct note_start_s *)p;
+          FAR struct trace_dump_task_context_s *tctxp;
+
+          tctxp = get_task_context(pid);
+          if (tctxp != NULL)
+            {
+              copy_task_name(tctxp->name, nst->nst_name);
+            }
+#endif
+
+          trace_dump_header(out, note);
+          fprintf(out, "sched_wakeup_new: comm=%s pid=%d target_cpu=%d\n",
+                  get_task_name(pid), get_pid(pid), cpu);
+        }
+        break;
+
+      case NOTE_STOP:
+        {
+          trace_dump_header(out, note);
+          fprintf(out, "sched_switch: "
+                       "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+                       "next_comm=%s next_pid=%u\n",
+                  get_task_name(pid), get_pid(pid), 'X',
+                  get_task_name(ctxp->current_pid),
+                  get_pid(ctxp->current_pid));
+        }
+        break;
+
+      case NOTE_SUSPEND:
+        {
+          FAR struct note_suspend_s *nsu = (FAR struct note_suspend_s *)p;
+
+          /* This note informs the task to be suspended.
+           * Preserve the information for the succeeding NOTE_RESUME.
+           */
+
+          ctxp->current_state = nsu->nsu_state;
+        }
+        break;
+
+      case NOTE_RESUME:
+        {
+          /* This note informs the task to be resumed.
+           * The task switch timing depends on the running context.
+           */
+
+          ctxp->next_pid = pid;
+
+          if (!ctxp->ininterrupt)
+            {
+              /* If not in the interrupt context, the task switch is
+               * executed immediately.
+               */
+
+              trace_dump_header(out, note);
+              trace_dump_sched_switch(out, note);
+            }
+          else
+            {
+              /* If in the interrupt context, the task switch is postponed
+               * until leaving the interrupt handler.
+               */
+
+              trace_dump_header(out, note);
+              fprintf(out, "sched_waking: comm=%s pid=%d target_cpu=%d\n",
+                      get_task_name(ctxp->next_pid),
+                      get_pid(ctxp->next_pid), cpu);
+              ctxp->pendingswitch = true;
+            }
+        }
+        break;
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      case NOTE_SYSCALL_ENTER:
+        {
+          FAR struct note_syscall_enter_s *nsc;
+          FAR struct trace_dump_task_context_s *tctxp;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (ctxp->ininterrupt)
+            {
+              break;
+            }
+
+          tctxp = get_task_context(pid);
+          if (tctxp == NULL)
+            {
+              break;
+            }
+
+          tctxp->syscall_nest++;
+          if (tctxp->syscall_nest > 1)
+            {
+              break;
+            }
+
+          nsc = (FAR struct note_syscall_enter_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "sys_%s()\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED]);
+        }
+        break;
+
+      case NOTE_SYSCALL_LEAVE:
+        {
+          FAR struct note_syscall_leave_s *nsc;
+          FAR struct trace_dump_task_context_s *tctxp;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (ctxp->ininterrupt)
+            {
+              break;
+            }
+
+          tctxp = get_task_context(pid);
+          if (tctxp == NULL)
+            {
+              break;
+            }
+
+          tctxp->syscall_nest--;
+          if (tctxp->syscall_nest > 0)
+            {
+              break;
+            }
+
+          tctxp->syscall_nest = 0;
+
+          nsc = (FAR struct note_syscall_leave_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "sys_%s -> 0x%x\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED],
+                  nsc->nsc_result);
+        }
+        break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+      case NOTE_IRQ_ENTER:
+        {
+          FAR struct note_irqhandler_s *nih;
+
+          nih = (FAR struct note_irqhandler_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "irq_handler_entry: irq=%u\n",
+                  nih->nih_irq);
+          ctxp->ininterrupt = true;
+        }
+        break;
+
+      case NOTE_IRQ_LEAVE:
+        {
+          FAR struct note_irqhandler_s *nih;
+
+          nih = (FAR struct note_irqhandler_s *)p;
+          trace_dump_header(out, note);
+          fprintf(out, "irq_handler_exit: irq=%u\n",
+                  nih->nih_irq);
+          ctxp->ininterrupt = false;
+          if (ctxp->pendingswitch)
+            {
+              /* If the pending task switch exists, it is executed here */
+
+              trace_dump_header(out, note);
+              trace_dump_sched_switch(out, note);
+            }
+        }
+        break;
+#endif
+
+      default:
+        break;
+    }
+
+  /* Return the length of the processed note */
+
+  return note->nc_length;
+}
+
+/****************************************************************************
+ * Name: trace_dump
+ ****************************************************************************/
+
+static int trace_dump(FILE *out)
+{
+  uint8_t tracedata[UCHAR_MAX];
+  FAR uint8_t *p;
+  int size;
+  int ret;
+  int fd;
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Open note for read */
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      return ERROR;
+    }
+
+  trace_dump_init_context();
+
+  /* Read and output all notes */
+
+  while (1)
+    {
+      ret = read(fd, tracedata, sizeof tracedata);
+      if (ret <= 0)
+        {
+          break;
+        }
+
+      p = tracedata;
+      do
+        {
+          size = trace_dump_one(out, p);
+          p += size;
+          ret -= size;
+        }
+      while (ret > 0);
+    }
+
+  trace_dump_fini_context();
+
+  /* Close note */
+
+  close(fd);
+
+  return ret;
+}
+
+#endif /* CONFIG_DRIVER_NOTERAM */
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(FAR struct nsh_vtbl_s *vtbl,
+                           int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          nsh_output(vtbl, g_fmtarginvalid, argv[0]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(FAR struct nsh_vtbl_s *vtbl,
+                          int index, int argc, char **argv)
+{
+  FAR char *fullpath;
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          fullpath = nsh_getfullpath(vtbl, argv[index]);
+          if (fullpath == NULL)
+            {
+              nsh_output(vtbl, g_fmtcmdoutofmemory, argv[0]);
+              return ERROR;
+            }
+
+          out = fopen(fullpath, "w");
+          if (out == NULL)
+            {
+              nsh_output(vtbl, g_fmtcmdfailed, argv[0], "open", NSH_ERRNO);
+              return ERROR;
+            }
+
+          nsh_freefullpath(fullpath);
+        }
+    }
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      nsh_output(vtbl, g_fmtcmdfailed, argv[0], "open", NSH_ERRNO);
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+static int trace_cmd_cmd(FAR struct nsh_vtbl_s *vtbl,
+                         int index, int argc, char **argv)
+{
+#ifndef CONFIG_NSH_DISABLEBG
+  bool bgsave;
+#endif
+#if CONFIG_FILE_STREAM
+  bool redirsave;
+#endif
+  int ret;
+
+  /* Usage: trace cmd "<command>" */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      nsh_output(vtbl, g_fmtargrequired, argv[0]);
+      return ERROR;
+    }
+
+  /* Save state */
+
+#ifndef CONFIG_NSH_DISABLEBG
+  bgsave    = vtbl->np.np_bg;
+#endif
+#if CONFIG_FILE_STREAM
+  redirsave = vtbl->np.np_redirect;
+#endif
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  ret = nsh_parse(vtbl, argv[index]);
+  notectl_enable(false);
+
+  /* Restore state */
+
+#ifndef CONFIG_NSH_DISABLEBG
+  vtbl->np.np_bg       = bgsave;
+#endif
+#if CONFIG_FILE_STREAM
+  vtbl->np.np_redirect = redirsave;
+#endif
+
+  if (ret < 0)
+    {
+      return ret;
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(FAR struct nsh_vtbl_s *vtbl,
+                          int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            nsh_output(vtbl, g_fmtsyntax, argv[0]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  nsh_output(vtbl, "Task trace mode:\n");
+  nsh_output(vtbl, " Trace                   : %s\n",
+             (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+              "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  nsh_output(vtbl, " Overwrite               : %s\n",
+             (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+              "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  nsh_output(vtbl, " Syscall trace           : %s\n",
+             mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+               "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      nsh_output(vtbl, "  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  nsh_output(vtbl, " IRQ trace               : %s\n",
+             mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+               "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      nsh_output(vtbl, "  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(FAR struct nsh_vtbl_s *vtbl,
+                            int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      nsh_output(vtbl, "Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              nsh_output(vtbl, "  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(FAR struct nsh_vtbl_s *vtbl,
+                         int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          nsh_output(vtbl, g_fmtarginvalid, argv[0]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      nsh_output(vtbl, "Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              nsh_output(vtbl, "  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: cmd_trace
+ ****************************************************************************/
+
+int cmd_trace(FAR struct nsh_vtbl_s *vtbl, int argc, char **argv)
+{
+  int ret = OK;
+  int i;
+
+  /* Open note control device */
+
+  g_notectlfd = open("/dev/notectl", 0);

Review comment:
       change g_notectlfd  to local variable and pass it to other function as argument?

##########
File path: nshlib/nsh_trace.c
##########
@@ -0,0 +1,1326 @@
+/****************************************************************************
+ * apps/nshlib/nsh_trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <time.h>
+#include <nuttx/clock.h>
+
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+#ifdef CONFIG_DRIVER_NOTECTL
+#  include <nuttx/note/notectl_driver.h>
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+#include "nsh.h"
+#include "nsh_console.h"
+
+#ifdef CONFIG_DRIVER_NOTECTL

Review comment:
       don't need? guard by Makefile already.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r502255318



##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       Ok.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503735096



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;

Review comment:
       Same as https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503734715




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r504389948



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>

Review comment:
       don't need?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static int notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  trace_dump_clear();

Review comment:
       should we let user clear by command manually? or at least by option like this:
   start -c

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static int notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  trace_dump_clear();
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true, notectlfd);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  FAR FILE *out = stdout;
+  unsigned int owmode;
+  bool enable;
+  int ret;
+
+  /* Usage: trace dump [{+|-}o] [<filename>] */
+
+  if (index == argc - 1)
+    {
+      /* No argument - display current overwrite mode */
+
+      owmode = trace_dump_getmode();
+      printf("Trace dump mode:\n");
+      printf(" Overwrite               : %s\n",
+             (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+              "on  (+o)" : "off (-o)");
+
+      return index;
+    }
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace dump: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      trace_dump_setmode(owmode);
+    }
+
+  if (index == argc - 1)
+    {
+      /* <filename> is not given - change overwrite mode only */
+
+      return index;
+    }
+
+  /* If <filename> is given, open the file stream for output */
+
+  index++;
+  out = fopen(argv[index], "w");
+  if (out == NULL)
+    {
+      fprintf(stderr,
+              "trace dump: cannot open '%s'\n", argv[index]);
+      return ERROR;
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false, notectlfd);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* Close the file stream for dump. */
+
+  fclose(out);
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, FAR char **argv, int notectlfd)
+{
+  char command[CONFIG_NSH_LINELEN];
+  int changed;
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  trace_dump_clear();
+#endif
+
+  /* Execute the command with tracing */
+
+  changed = notectl_enable(true, notectlfd);
+
+  system(command);
+
+  if (changed)
+    {
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  for (count = i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  for (count = i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, FAR char **argv,
+                             int notectlfd)
+{
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      for (count = n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, FAR char **argv, int notectlfd)
+{
+  bool enable;
+  bool modified = false;
+  int irqno;
+  FAR char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      for (count = n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(void)
+{
+  fprintf(stderr,
+          "\nUsage: trace [<subcommand>...]\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [{+|-}o] [<filename>]      :"
+                                " Output the trace result\n"
+#endif
+          "  mode [{+|-}{s|i}...]            :"
+                                " Set task trace options\n"
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          "  syscall [{+|-}<syscallname>...] :"
+                                " Configure syscall trace filter\n"
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          "  irq [{+|-}<irqnum>...]          :"
+                                " Configure IRQ trace filter\n"
+#endif
+          "\nIf <subcommand> is not specified, output the trace result "
+          "to stdout.\n"
+         );
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+int main(int argc, FAR char *argv[])
+{
+  int notectlfd;
+  int exitcode = EXIT_FAILURE;
+  int i;
+
+  /* Open note control device */
+
+  notectlfd = open("/dev/notectl", 0);
+  if (notectlfd < 0)
+    {
+      fprintf(stderr,
+              "trace: cannot open /dev/notectl\n");
+      goto errout;
+    }
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (argc == 1)
+    {
+      /* No arguments - dump the trace data to stdout */
+
+      notectl_enable(false, notectlfd);

Review comment:
       should we restore the state or don't change the state at all?

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p,
+                          FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      cctx->current_pid = pid;
+    }
+
+  /* Output one note */
+
+  switch (note->nc_type)
+    {
+      case NOTE_START:
+        {
+#if CONFIG_TASK_NAME_SIZE > 0
+          FAR struct note_start_s *nst = (FAR struct note_start_s *)p;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx != NULL)
+            {
+              copy_task_name(tctx->name, nst->nst_name);
+            }
+#endif
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_wakeup_new: comm=%s pid=%d target_cpu=%d\n",
+                  get_task_name(pid, ctx), get_pid(pid), cpu);
+        }
+        break;
+
+      case NOTE_STOP:
+        {
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_switch: "
+                       "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+                       "next_comm=%s next_pid=%u\n",
+                  get_task_name(pid, ctx), get_pid(pid), 'X',
+                  get_task_name(cctx->current_pid, ctx),
+                  get_pid(cctx->current_pid));
+        }
+        break;
+
+      case NOTE_SUSPEND:
+        {
+          FAR struct note_suspend_s *nsu = (FAR struct note_suspend_s *)p;
+
+          /* This note informs the task to be suspended.
+           * Preserve the information for the succeeding NOTE_RESUME.
+           */
+
+          cctx->current_state = nsu->nsu_state;
+        }
+        break;
+
+      case NOTE_RESUME:
+        {
+          /* This note informs the task to be resumed.
+           * The task switch timing depends on the running context.
+           */
+
+          cctx->next_pid = pid;
+
+          if (!cctx->ininterrupt)
+            {
+              /* If not in the interrupt context, the task switch is
+               * executed immediately.
+               */
+
+              trace_dump_header(out, note, ctx);
+              trace_dump_sched_switch(out, note, ctx);
+            }
+          else
+            {
+              /* If in the interrupt context, the task switch is postponed
+               * until leaving the interrupt handler.
+               */
+
+              trace_dump_header(out, note, ctx);
+              fprintf(out, "sched_waking: comm=%s pid=%d target_cpu=%d\n",
+                      get_task_name(cctx->next_pid, ctx),
+                      get_pid(cctx->next_pid), cpu);
+              cctx->pendingswitch = true;
+            }
+        }
+        break;
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      case NOTE_SYSCALL_ENTER:
+        {
+          FAR struct note_syscall_enter_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest++;
+          if (tctx->syscall_nest > 1)
+            {
+              break;
+            }
+
+          nsc = (FAR struct note_syscall_enter_s *)p;
+          if (nsc->nsc_nr < CONFIG_SYS_RESERVED ||
+              nsc->nsc_nr >= SYS_maxsyscall)
+            {
+              break;
+            }
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sys_%s()\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED]);
+        }
+        break;
+
+      case NOTE_SYSCALL_LEAVE:
+        {
+          FAR struct note_syscall_leave_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest--;
+          if (tctx->syscall_nest > 0)
+            {
+              break;
+            }
+
+          tctx->syscall_nest = 0;

Review comment:
       don't need

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static int notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  trace_dump_clear();
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true, notectlfd);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  FAR FILE *out = stdout;
+  unsigned int owmode;
+  bool enable;
+  int ret;
+
+  /* Usage: trace dump [{+|-}o] [<filename>] */
+
+  if (index == argc - 1)
+    {
+      /* No argument - display current overwrite mode */
+
+      owmode = trace_dump_getmode();
+      printf("Trace dump mode:\n");
+      printf(" Overwrite               : %s\n",
+             (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+              "on  (+o)" : "off (-o)");
+
+      return index;
+    }
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace dump: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      trace_dump_setmode(owmode);
+    }
+
+  if (index == argc - 1)
+    {
+      /* <filename> is not given - change overwrite mode only */
+
+      return index;
+    }
+
+  /* If <filename> is given, open the file stream for output */
+
+  index++;
+  out = fopen(argv[index], "w");
+  if (out == NULL)
+    {
+      fprintf(stderr,
+              "trace dump: cannot open '%s'\n", argv[index]);
+      return ERROR;
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false, notectlfd);

Review comment:
       should we restore the state?

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p,
+                          FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      cctx->current_pid = pid;

Review comment:
       don't really need?

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,

Review comment:
       add FAR for all FILE *

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p,
+                          FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      cctx->current_pid = pid;
+    }
+
+  /* Output one note */
+
+  switch (note->nc_type)
+    {
+      case NOTE_START:
+        {
+#if CONFIG_TASK_NAME_SIZE > 0
+          FAR struct note_start_s *nst = (FAR struct note_start_s *)p;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx != NULL)
+            {
+              copy_task_name(tctx->name, nst->nst_name);
+            }
+#endif
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_wakeup_new: comm=%s pid=%d target_cpu=%d\n",
+                  get_task_name(pid, ctx), get_pid(pid), cpu);
+        }
+        break;
+
+      case NOTE_STOP:
+        {
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_switch: "
+                       "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+                       "next_comm=%s next_pid=%u\n",
+                  get_task_name(pid, ctx), get_pid(pid), 'X',
+                  get_task_name(cctx->current_pid, ctx),
+                  get_pid(cctx->current_pid));
+        }
+        break;
+
+      case NOTE_SUSPEND:
+        {
+          FAR struct note_suspend_s *nsu = (FAR struct note_suspend_s *)p;
+
+          /* This note informs the task to be suspended.
+           * Preserve the information for the succeeding NOTE_RESUME.
+           */
+
+          cctx->current_state = nsu->nsu_state;
+        }
+        break;
+
+      case NOTE_RESUME:
+        {
+          /* This note informs the task to be resumed.
+           * The task switch timing depends on the running context.
+           */
+
+          cctx->next_pid = pid;
+
+          if (!cctx->ininterrupt)
+            {
+              /* If not in the interrupt context, the task switch is
+               * executed immediately.
+               */
+
+              trace_dump_header(out, note, ctx);
+              trace_dump_sched_switch(out, note, ctx);
+            }
+          else
+            {
+              /* If in the interrupt context, the task switch is postponed
+               * until leaving the interrupt handler.
+               */
+
+              trace_dump_header(out, note, ctx);
+              fprintf(out, "sched_waking: comm=%s pid=%d target_cpu=%d\n",
+                      get_task_name(cctx->next_pid, ctx),
+                      get_pid(cctx->next_pid), cpu);
+              cctx->pendingswitch = true;
+            }
+        }
+        break;
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      case NOTE_SYSCALL_ENTER:
+        {
+          FAR struct note_syscall_enter_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest++;
+          if (tctx->syscall_nest > 1)

Review comment:
       should we always add syscall before check ininterrupt flag? otherwise the count may mismatch.
   }

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);

Review comment:
       But it can't avoid in protect/kernel build anyway, should we keep the consistence?

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;

Review comment:
       change to?
   ctx->cpu[cpu].current_pid = cpu;

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,751 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/lib/regex.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static int notectl_enable(int flag, int notectlfd)
+{
+  struct note_filter_mode_s mode;
+  int oldflag;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  oldflag = (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) != 0;
+  if (flag == oldflag)
+    {
+      /* Already set */
+
+      return false;
+    }
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+
+  return true;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, FAR char **argv,
+                           int notectlfd)
+{
+  FAR char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  trace_dump_clear();
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true, notectlfd);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false, notectlfd);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, FAR char **argv,
+                          int notectlfd)
+{
+  FAR FILE *out = stdout;
+  unsigned int owmode;
+  bool enable;
+  int ret;
+
+  /* Usage: trace dump [{+|-}o] [<filename>] */
+
+  if (index == argc - 1)
+    {
+      /* No argument - display current overwrite mode */
+
+      owmode = trace_dump_getmode();
+      printf("Trace dump mode:\n");
+      printf(" Overwrite               : %s\n",
+             (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+              "on  (+o)" : "off (-o)");
+
+      return index;
+    }
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace dump: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      trace_dump_setmode(owmode);
+    }
+
+  if (index == argc - 1)
+    {
+      /* <filename> is not given - change overwrite mode only */
+
+      return index;
+    }
+
+  /* If <filename> is given, open the file stream for output */
+
+  index++;
+  out = fopen(argv[index], "w");
+  if (out == NULL)
+    {
+      fprintf(stderr,
+              "trace dump: cannot open '%s'\n", argv[index]);
+      return ERROR;
+    }
+
+  /* Stop the tracing before dump */

Review comment:
       should we add an option to bypass the disable?

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p,
+                          FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      cctx->current_pid = pid;
+    }
+
+  /* Output one note */
+
+  switch (note->nc_type)
+    {
+      case NOTE_START:
+        {
+#if CONFIG_TASK_NAME_SIZE > 0
+          FAR struct note_start_s *nst = (FAR struct note_start_s *)p;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx != NULL)
+            {
+              copy_task_name(tctx->name, nst->nst_name);
+            }
+#endif
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_wakeup_new: comm=%s pid=%d target_cpu=%d\n",
+                  get_task_name(pid, ctx), get_pid(pid), cpu);
+        }
+        break;
+
+      case NOTE_STOP:
+        {
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_switch: "
+                       "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+                       "next_comm=%s next_pid=%u\n",
+                  get_task_name(pid, ctx), get_pid(pid), 'X',
+                  get_task_name(cctx->current_pid, ctx),
+                  get_pid(cctx->current_pid));
+        }
+        break;
+
+      case NOTE_SUSPEND:
+        {
+          FAR struct note_suspend_s *nsu = (FAR struct note_suspend_s *)p;
+
+          /* This note informs the task to be suspended.
+           * Preserve the information for the succeeding NOTE_RESUME.
+           */
+
+          cctx->current_state = nsu->nsu_state;
+        }
+        break;
+
+      case NOTE_RESUME:
+        {
+          /* This note informs the task to be resumed.
+           * The task switch timing depends on the running context.
+           */
+
+          cctx->next_pid = pid;
+
+          if (!cctx->ininterrupt)
+            {
+              /* If not in the interrupt context, the task switch is
+               * executed immediately.
+               */
+
+              trace_dump_header(out, note, ctx);
+              trace_dump_sched_switch(out, note, ctx);
+            }
+          else
+            {
+              /* If in the interrupt context, the task switch is postponed
+               * until leaving the interrupt handler.
+               */
+
+              trace_dump_header(out, note, ctx);
+              fprintf(out, "sched_waking: comm=%s pid=%d target_cpu=%d\n",
+                      get_task_name(cctx->next_pid, ctx),
+                      get_pid(cctx->next_pid), cpu);
+              cctx->pendingswitch = true;
+            }
+        }
+        break;
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      case NOTE_SYSCALL_ENTER:
+        {
+          FAR struct note_syscall_enter_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest++;
+          if (tctx->syscall_nest > 1)
+            {
+              break;
+            }
+
+          nsc = (FAR struct note_syscall_enter_s *)p;
+          if (nsc->nsc_nr < CONFIG_SYS_RESERVED ||
+              nsc->nsc_nr >= SYS_maxsyscall)
+            {
+              break;
+            }
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sys_%s()\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED]);
+        }
+        break;
+
+      case NOTE_SYSCALL_LEAVE:
+        {
+          FAR struct note_syscall_leave_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest--;
+          if (tctx->syscall_nest > 0)
+            {
+              break;
+            }
+
+          tctx->syscall_nest = 0;
+
+          nsc = (FAR struct note_syscall_leave_s *)p;
+          if (nsc->nsc_nr < CONFIG_SYS_RESERVED ||
+              nsc->nsc_nr >= SYS_maxsyscall)
+            {
+              break;
+            }
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sys_%s -> 0x%x\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED],
+                  nsc->nsc_result);
+        }
+        break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+      case NOTE_IRQ_ENTER:
+        {
+          FAR struct note_irqhandler_s *nih;
+
+          nih = (FAR struct note_irqhandler_s *)p;
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "irq_handler_entry: irq=%u\n",
+                  nih->nih_irq);
+          cctx->ininterrupt = true;

Review comment:
       should we use count here?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#issuecomment-709308221


   @xiaoxiang781216 Thank you for your great support for my kernel and app PRs!
   I'll work on the maintenance of the documents in https://github.com/apache/incubator-nuttx/pull/1955 .
   


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 merged pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 merged pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421


   


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503970834



##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,570 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <nuttx/sched_note.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+static FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;

Review comment:
       I have withdrawn the last comment https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503735462 and changed to local variable in e2a8784.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r505123470



##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p,
+                          FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      cctx->current_pid = pid;
+    }
+
+  /* Output one note */
+
+  switch (note->nc_type)
+    {
+      case NOTE_START:
+        {
+#if CONFIG_TASK_NAME_SIZE > 0
+          FAR struct note_start_s *nst = (FAR struct note_start_s *)p;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx != NULL)
+            {
+              copy_task_name(tctx->name, nst->nst_name);
+            }
+#endif
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_wakeup_new: comm=%s pid=%d target_cpu=%d\n",
+                  get_task_name(pid, ctx), get_pid(pid), cpu);
+        }
+        break;
+
+      case NOTE_STOP:
+        {
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_switch: "
+                       "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+                       "next_comm=%s next_pid=%u\n",
+                  get_task_name(pid, ctx), get_pid(pid), 'X',
+                  get_task_name(cctx->current_pid, ctx),
+                  get_pid(cctx->current_pid));
+        }
+        break;
+
+      case NOTE_SUSPEND:
+        {
+          FAR struct note_suspend_s *nsu = (FAR struct note_suspend_s *)p;
+
+          /* This note informs the task to be suspended.
+           * Preserve the information for the succeeding NOTE_RESUME.
+           */
+
+          cctx->current_state = nsu->nsu_state;
+        }
+        break;
+
+      case NOTE_RESUME:
+        {
+          /* This note informs the task to be resumed.
+           * The task switch timing depends on the running context.
+           */
+
+          cctx->next_pid = pid;
+
+          if (!cctx->ininterrupt)
+            {
+              /* If not in the interrupt context, the task switch is
+               * executed immediately.
+               */
+
+              trace_dump_header(out, note, ctx);
+              trace_dump_sched_switch(out, note, ctx);
+            }
+          else
+            {
+              /* If in the interrupt context, the task switch is postponed
+               * until leaving the interrupt handler.
+               */
+
+              trace_dump_header(out, note, ctx);
+              fprintf(out, "sched_waking: comm=%s pid=%d target_cpu=%d\n",
+                      get_task_name(cctx->next_pid, ctx),
+                      get_pid(cctx->next_pid), cpu);
+              cctx->pendingswitch = true;
+            }
+        }
+        break;
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      case NOTE_SYSCALL_ENTER:
+        {
+          FAR struct note_syscall_enter_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest++;
+          if (tctx->syscall_nest > 1)
+            {
+              break;
+            }
+
+          nsc = (FAR struct note_syscall_enter_s *)p;
+          if (nsc->nsc_nr < CONFIG_SYS_RESERVED ||
+              nsc->nsc_nr >= SYS_maxsyscall)
+            {
+              break;
+            }
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sys_%s()\n",
+                  g_funcnames[nsc->nsc_nr - CONFIG_SYS_RESERVED]);
+        }
+        break;
+
+      case NOTE_SYSCALL_LEAVE:
+        {
+          FAR struct note_syscall_leave_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest--;
+          if (tctx->syscall_nest > 0)
+            {
+              break;
+            }
+
+          tctx->syscall_nest = 0;

Review comment:
       It is needed because syscall enter and leave in the trace data are not always coupled.
   If only syscall enter is overwritten and lost in the trace data, the first syscall leave may make nest to negative value.

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,643 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/sched_note.h>
+#include <nuttx/note/noteram_driver.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+struct trace_dump_context_s
+{
+  struct trace_dump_cpu_context_s cpu[NCPUS];
+  FAR struct trace_dump_task_context_s *task;
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int notefd;
+
+  notefd = open("/dev/note", O_RDONLY);
+  if (notefd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(notefd, cmd, arg);
+  close(notefd);
+}
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(FAR struct trace_dump_context_s *ctx)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      ctx->cpu[cpu].ininterrupt = false;
+      ctx->cpu[cpu].pendingswitch = false;
+      ctx->cpu[cpu].current_state = TSTATE_TASK_RUNNING;
+      ctx->cpu[cpu].current_pid = 0;
+      ctx->cpu[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = ctx->task;
+  ctx->task = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid,
+                                      FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &ctx->task;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid,
+                                 FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+
+  tctx = get_task_context(pid, ctx);
+  if (tctx != NULL && tctx->name[0] != '\0')
+    {
+      return tctx->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note,
+                              FAR struct trace_dump_context_s *ctx)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = ctx->cpu[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid, ctx), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);
+}
+
+/****************************************************************************
+ * Name: trace_dump_sched_switch
+ ****************************************************************************/
+
+static void trace_dump_sched_switch(FILE *out,
+                                    FAR struct note_common_s *note,
+                                    FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t current_pid;
+  pid_t next_pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  current_pid = cctx->current_pid;
+  next_pid = cctx->next_pid;
+
+  fprintf(out, "sched_switch: "
+               "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+               "next_comm=%s next_pid=%u\n",
+          get_task_name(current_pid, ctx), get_pid(current_pid),
+          get_task_state(cctx->current_state),
+          get_task_name(next_pid, ctx), get_pid(next_pid));
+
+  cctx->current_pid = cctx->next_pid;
+  cctx->pendingswitch = false;
+}
+
+/****************************************************************************
+ * Name: trace_dump_one
+ ****************************************************************************/
+
+static int trace_dump_one(FILE *out,
+                          FAR uint8_t *p,
+                          FAR struct trace_dump_context_s *ctx)
+{
+  FAR struct note_common_s *note = (FAR struct note_common_s *)p;
+  FAR struct trace_dump_cpu_context_s *cctx;
+  pid_t pid;
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  cctx = &ctx->cpu[cpu];
+  pid = note->nc_pid[0] + (note->nc_pid[1] << 8);
+
+  if (note->nc_type != NOTE_START &&
+      note->nc_type != NOTE_STOP &&
+      note->nc_type != NOTE_RESUME
+#ifdef CONFIG_SMP
+      && !(note->nc_type >= NOTE_CPU_START &&
+           note->nc_type <= NOTE_CPU_RESUMED)
+#endif
+     )
+    {
+      cctx->current_pid = pid;
+    }
+
+  /* Output one note */
+
+  switch (note->nc_type)
+    {
+      case NOTE_START:
+        {
+#if CONFIG_TASK_NAME_SIZE > 0
+          FAR struct note_start_s *nst = (FAR struct note_start_s *)p;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx != NULL)
+            {
+              copy_task_name(tctx->name, nst->nst_name);
+            }
+#endif
+
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_wakeup_new: comm=%s pid=%d target_cpu=%d\n",
+                  get_task_name(pid, ctx), get_pid(pid), cpu);
+        }
+        break;
+
+      case NOTE_STOP:
+        {
+          trace_dump_header(out, note, ctx);
+          fprintf(out, "sched_switch: "
+                       "prev_comm=%s prev_pid=%u prev_state=%c ==> "
+                       "next_comm=%s next_pid=%u\n",
+                  get_task_name(pid, ctx), get_pid(pid), 'X',
+                  get_task_name(cctx->current_pid, ctx),
+                  get_pid(cctx->current_pid));
+        }
+        break;
+
+      case NOTE_SUSPEND:
+        {
+          FAR struct note_suspend_s *nsu = (FAR struct note_suspend_s *)p;
+
+          /* This note informs the task to be suspended.
+           * Preserve the information for the succeeding NOTE_RESUME.
+           */
+
+          cctx->current_state = nsu->nsu_state;
+        }
+        break;
+
+      case NOTE_RESUME:
+        {
+          /* This note informs the task to be resumed.
+           * The task switch timing depends on the running context.
+           */
+
+          cctx->next_pid = pid;
+
+          if (!cctx->ininterrupt)
+            {
+              /* If not in the interrupt context, the task switch is
+               * executed immediately.
+               */
+
+              trace_dump_header(out, note, ctx);
+              trace_dump_sched_switch(out, note, ctx);
+            }
+          else
+            {
+              /* If in the interrupt context, the task switch is postponed
+               * until leaving the interrupt handler.
+               */
+
+              trace_dump_header(out, note, ctx);
+              fprintf(out, "sched_waking: comm=%s pid=%d target_cpu=%d\n",
+                      get_task_name(cctx->next_pid, ctx),
+                      get_pid(cctx->next_pid), cpu);
+              cctx->pendingswitch = true;
+            }
+        }
+        break;
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      case NOTE_SYSCALL_ENTER:
+        {
+          FAR struct note_syscall_enter_s *nsc;
+          FAR struct trace_dump_task_context_s *tctx;
+
+          /* Exclude the case of syscall issued by an interrupt handler and
+           * nested syscalls to correct tracecompass display.
+           */
+
+          if (cctx->ininterrupt)
+            {
+              break;
+            }
+
+          tctx = get_task_context(pid, ctx);
+          if (tctx == NULL)
+            {
+              break;
+            }
+
+          tctx->syscall_nest++;
+          if (tctx->syscall_nest > 1)

Review comment:
       The syscall nest while executing the interrupt handler should not belong to the syscall nest in the task.
   For now, because all syscalls in the interrupt handler cannot be displayed, we can ignore the syscall immediately if in the interrupt handler.
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503734916



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"
+                                " Output the trace result\n"
+#endif
+          "  mode [{+|-}{o|s|i}...]          :"
+                                " Set task trace options\n"
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          "  syscall [{+|-}<syscallname>...] :"
+                                " Configure syscall trace filter\n"
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          "  irq [{+|-}<irqnum>...]          :"
+                                " Configure IRQ trace filter\n"
+#endif
+         );
+  exit(exitcode);
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+int main(int argc, FAR char *argv[])
+{
+  int exitcode;
+  int i;
+
+  exitcode = EXIT_FAILURE;
+
+  /* Open note control device */
+
+  g_notectlfd = open("/dev/notectl", 0);
+  if (g_notectlfd < 0)
+    {
+      fprintf(stderr,
+              "trace: cannot open /dev/notectl\n");
+      goto errout;
+    }
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (argc == 1)
+    {
+      /* No arguments - dump the trace data */
+
+      notectl_enable(false);

Review comment:
       Same as https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503734494




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503734494



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);

Review comment:
       trace_dump() continues dumping until all data in the buffer is dumped. 
   Because trace_dump() itself issues syscalls, if the trace is not stopped before dump, these are added in the buffer while dumping. So, once the dump is started, it never stops.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r505144232



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)

Review comment:
       How about we change trace_dump_get_mode to trace_dump_get_overwrite or trace_dump_is_overwrite? so true/false is used in trace.c and trace_dump.c convert true/false to NOTERAM_MODE_OVERWRITE?




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r501507319



##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       To say the truth, I troubled what command name should be given to this feature.
   As you mentioned, this feature strongly depends on note of the kernel function, but "note" command already exists at apps/system/sched_note, and from the user point of view, the feature realized by this command is truly task trace.
   So I decided to name "trace" to this feature.
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503735633



##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,570 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <nuttx/sched_note.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+static FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(void)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      g_dump_cpu_ctx[cpu].ininterrupt = false;
+      g_dump_cpu_ctx[cpu].pendingswitch = false;
+      g_dump_cpu_ctx[cpu].current_state = TSTATE_TASK_RUNNING;
+      g_dump_cpu_ctx[cpu].current_pid = 0;
+      g_dump_cpu_ctx[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(void)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = g_dump_task_ctx;
+  g_dump_task_ctx = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &g_dump_task_ctx;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s *tctxp;
+
+  tctxp = get_task_context(pid);
+  if (tctxp != NULL && tctxp->name[0] != '\0')
+    {
+      return tctxp->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = g_dump_cpu_ctx[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);

Review comment:
       `systime % (1000 * 1000 / CONFIG_USEC_PER_TICK)` gets the number of ticks less than a second.
   Because it is a unit of ticks, we need to multiply `CONFIG_USEC_PER_TICK * 1000` to get a unit of nanosecs which the ftrace format requires.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r504477083



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"

Review comment:
       Though once I have tried to fix "trace dump" command to follow your proposals, I have reverted the changes.
   Especially, the following syntax seems strange to me.
   trace -> dump trace data to console
   trace dump ->show the overwrite state
   trace dump filename->save to file
   
   I have reverted it and changed the default behavior of trace command (without options).
   trace -> show current state (same as "trace mode")
   trace dump -> dump trace data to console
   trace dump filename -> save to file (dump trace data to file)
   How about it?
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#issuecomment-705990528


   The last commit 7e3a54a is a very trivial changes. Only README.md was changed.


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#issuecomment-705261540


   > I don't believe that this should be a part of NSH. NSH is a thin shell, like Bash, with a few simple built-in functions but its primary purpose is to interact with the system and with other programs. It should NOT contain any specialized programs itself. Any specialized programs must be moved to apps/system as built-in applications. There is no difference from the user experience, but this properly prevents NSH from exploding in size with specialized, programs. It must stay lean.
   
   @patacongo I understood. I'll try to fix it to built-in apps.


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503735462



##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,570 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <nuttx/sched_note.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+static FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;

Review comment:
       You mean both g_dump_cpu_ctx[] and g_dump_task_ctx?
   It seems a bit complicated to add both argument to every private functions.
   I think we have no strong reason to change into local variable because:
   - These are widely accessed by private functions in this file, 
   - Already defined as static and used only in this file.
   - Basically not called by multiple threads.
   




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] patacongo commented on pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
patacongo commented on pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#issuecomment-704923872


   I don't believe that this should be a part of NSH.  NSH is a thin shell, like Bash, with a few simple built-in functions but its primary purpose is to interact with the system and with other programs.  It should NOT contain any specialized programs itself.  Any specialized programs must be moved to apps/system as built-in applications.  There is no difference from the user experience, but this properly prevents NSH from exploding in size with specialized, programs.  It must stay lean.


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r501507319



##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       To say the truth, I troubled what command name should be given to this feature.
   As you mentioned, this feature strongly depends on note of the kernel function, but "note" command already exists at apps/system/sched_note, and from the user point of view, the feature realized by this command is truly task trace.
   So I decided to name "trace" to this feature.
   

##########
File path: nshlib/nsh_command.c
##########
@@ -518,6 +518,18 @@ static const struct cmdmap_s g_cmdmap[] =
   { "time",     cmd_time,     2, 2, "\"<command>\"" },
 #endif
 
+#ifdef CONFIG_DRIVER_NOTECTL
+# ifndef CONFIG_NSH_DISABLE_TRACE
+  { "trace",    cmd_trace,    1, CONFIG_NSH_MAXARGUMENTS,

Review comment:
       sched_note shows the scheduler notes as it is and I think it is important for kernel development, especially for scheduler.
   Different to it, my command processes and replaces the notes to fit for tracecompass. Moreover, basically my command intends to handle the notes after the trace is stopped. This is a different behavior to sched_note. So I'm negative to merge them into one.
   sched_note is still working. My changes for note device doesn't affect the existing behavior.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] YuuichiNakamura commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
YuuichiNakamura commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r503734715



##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;

Review comment:
       I got it. I have changed to let user invoke mode command manually.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



[GitHub] [incubator-nuttx-apps] xiaoxiang781216 commented on a change in pull request #421: Add task trace support

Posted by GitBox <gi...@apache.org>.
xiaoxiang781216 commented on a change in pull request #421:
URL: https://github.com/apache/incubator-nuttx-apps/pull/421#discussion_r502535726



##########
File path: system/trace/trace.h
##########
@@ -0,0 +1,59 @@
+/****************************************************************************
+ * apps/system/trace/trace.h
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+#ifndef __APPS_SYSTEM_TRACE_TRACE_H
+#define __APPS_SYSTEM_TRACE_TRACE_H
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+
+#ifdef __cplusplus
+#define EXTERN extern "C"
+extern "C"
+{
+#else
+#define EXTERN extern
+#endif
+
+/****************************************************************************
+ * Public Function Prototypes
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: trace_dump
+ *
+ * Description:
+ *   Read notes and dump trace results.
+ *
+ ****************************************************************************/
+
+int trace_dump(FILE *out);

Review comment:
       guard with CONFIG_DRIVER_NOTERAM too?

##########
File path: system/trace/Makefile
##########
@@ -0,0 +1,33 @@
+############################################################################
+# apps/system/trace/Makefile
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.  The
+# ASF licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+############################################################################
+
+include $(APPDIR)/Make.defs
+
+# trace command
+
+PROGNAME = $(CONFIG_SYSTEM_TRACE_PROGNAME)
+PRIORITY = $(CONFIG_SYSTEM_TRACE_PRIORITY)
+STACKSIZE = $(CONFIG_SYSTEM_TRACE_STACKSIZE)
+MODULE = $(CONFIG_SYSTEM_TRACE)
+
+CSRCS = trace_dump.c

Review comment:
       should we guard it with CONFIG_DRIVER_NOTERAM?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)

Review comment:
       change to for (count = i = 0;?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)

Review comment:
       change to for (count = i = 0;?

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,570 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <nuttx/sched_note.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+static FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;

Review comment:
       let's change to local variable?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"
+                                " Output the trace result\n"
+#endif
+          "  mode [{+|-}{o|s|i}...]          :"
+                                " Set task trace options\n"
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          "  syscall [{+|-}<syscallname>...] :"
+                                " Configure syscall trace filter\n"
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          "  irq [{+|-}<irqnum>...]          :"
+                                " Configure IRQ trace filter\n"
+#endif
+         );
+  exit(exitcode);
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+int main(int argc, FAR char *argv[])
+{
+  int exitcode;
+  int i;
+
+  exitcode = EXIT_FAILURE;
+
+  /* Open note control device */
+
+  g_notectlfd = open("/dev/notectl", 0);
+  if (g_notectlfd < 0)
+    {
+      fprintf(stderr,
+              "trace: cannot open /dev/notectl\n");
+      goto errout;
+    }
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (argc == 1)
+    {
+      /* No arguments - dump the trace data */
+
+      notectl_enable(false);
+      trace_dump(stdout);
+    }
+#endif
+
+  /* Parse command line arguments */
+
+  for (i = 1; i < argc; i++)
+    {
+      if (strcmp(argv[i], "start") == 0)
+        {
+          i = trace_cmd_start(i, argc, argv);
+        }
+      else if (strcmp(argv[i], "stop") == 0)
+        {
+          notectl_enable(false);
+        }
+#ifdef CONFIG_DRIVER_NOTERAM
+      else if (strcmp(argv[i], "dump") == 0)
+        {
+          i = trace_cmd_dump(i, argc, argv);
+        }
+#endif
+#ifdef CONFIG_SYSTEM_SYSTEM
+      else if (strcmp(argv[i], "cmd") == 0)
+        {
+          i = trace_cmd_cmd(i, argc, argv);
+        }
+#endif
+      else if (strcmp(argv[i], "mode") == 0)
+        {
+          i = trace_cmd_mode(i, argc, argv);
+        }
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      else if (strcmp(argv[i], "syscall") == 0)
+        {
+          i = trace_cmd_syscall(i, argc, argv);
+        }
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+      else if (strcmp(argv[i], "irq") == 0)
+        {
+          i = trace_cmd_irq(i, argc, argv);
+        }
+#endif
+      else
+        {
+          show_usage(argv[0], exitcode);
+          return exitcode;
+        }
+
+      if (i < 0)
+        {
+          break;

Review comment:
       return error here?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);

Review comment:
       why need stop trace?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)

Review comment:
       let's move all note related operation to trace_dump.c? and remove line 35-37. we can add more specific function e.g. trace_dump_clear, trace_dump_get_mode....

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);

Review comment:
       let's pass g_notectlfd as notectl_enable's argument to avoid the global variable?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);

Review comment:
       why not always call ioctl?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */

Review comment:
       change to local variable and pass to each funtion as argument?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;

Review comment:
       should we clear flag if all mask is clear? or let user invoke mode command manually?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"
+                                " Output the trace result\n"
+#endif
+          "  mode [{+|-}{o|s|i}...]          :"
+                                " Set task trace options\n"
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          "  syscall [{+|-}<syscallname>...] :"
+                                " Configure syscall trace filter\n"
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          "  irq [{+|-}<irqnum>...]          :"
+                                " Configure IRQ trace filter\n"
+#endif
+         );
+  exit(exitcode);
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+int main(int argc, FAR char *argv[])
+{
+  int exitcode;
+  int i;
+
+  exitcode = EXIT_FAILURE;
+
+  /* Open note control device */
+
+  g_notectlfd = open("/dev/notectl", 0);
+  if (g_notectlfd < 0)
+    {
+      fprintf(stderr,
+              "trace: cannot open /dev/notectl\n");
+      goto errout;
+    }
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (argc == 1)
+    {
+      /* No arguments - dump the trace data */
+
+      notectl_enable(false);
+      trace_dump(stdout);
+    }
+#endif
+
+  /* Parse command line arguments */
+
+  for (i = 1; i < argc; i++)
+    {
+      if (strcmp(argv[i], "start") == 0)
+        {
+          i = trace_cmd_start(i, argc, argv);
+        }
+      else if (strcmp(argv[i], "stop") == 0)
+        {
+          notectl_enable(false);
+        }
+#ifdef CONFIG_DRIVER_NOTERAM
+      else if (strcmp(argv[i], "dump") == 0)
+        {
+          i = trace_cmd_dump(i, argc, argv);
+        }
+#endif
+#ifdef CONFIG_SYSTEM_SYSTEM
+      else if (strcmp(argv[i], "cmd") == 0)
+        {
+          i = trace_cmd_cmd(i, argc, argv);
+        }
+#endif
+      else if (strcmp(argv[i], "mode") == 0)
+        {
+          i = trace_cmd_mode(i, argc, argv);
+        }
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+      else if (strcmp(argv[i], "syscall") == 0)
+        {
+          i = trace_cmd_syscall(i, argc, argv);
+        }
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+      else if (strcmp(argv[i], "irq") == 0)
+        {
+          i = trace_cmd_irq(i, argc, argv);
+        }
+#endif
+      else
+        {
+          show_usage(argv[0], exitcode);

Review comment:
       g_notectlfd not close? or remove line 806 and let os close it for us.

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)

Review comment:
       call match in nuttx\include\nuttx\lib\regex.h directly?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"
+                                " Output the trace result\n"
+#endif
+          "  mode [{+|-}{o|s|i}...]          :"
+                                " Set task trace options\n"
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          "  syscall [{+|-}<syscallname>...] :"
+                                " Configure syscall trace filter\n"
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          "  irq [{+|-}<irqnum>...]          :"
+                                " Configure IRQ trace filter\n"
+#endif
+         );
+  exit(exitcode);
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+int main(int argc, FAR char *argv[])
+{
+  int exitcode;
+  int i;
+
+  exitcode = EXIT_FAILURE;
+
+  /* Open note control device */
+
+  g_notectlfd = open("/dev/notectl", 0);
+  if (g_notectlfd < 0)
+    {
+      fprintf(stderr,
+              "trace: cannot open /dev/notectl\n");
+      goto errout;
+    }
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  if (argc == 1)
+    {
+      /* No arguments - dump the trace data */
+
+      notectl_enable(false);

Review comment:
       why disable the trace?

##########
File path: system/trace/trace_dump.c
##########
@@ -0,0 +1,570 @@
+/****************************************************************************
+ * apps/system/trace/trace_dump.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <nuttx/sched_note.h>
+
+#include "trace.h"
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+#  ifdef CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#  else
+#    define CONFIG_LIB_SYSCALL
+#    include <syscall.h>
+#    undef CONFIG_LIB_SYSCALL
+#  endif
+#endif
+
+#ifdef CONFIG_SMP
+#  define NCPUS CONFIG_SMP_NCPUS
+#else
+#  define NCPUS 1
+#endif
+
+/****************************************************************************
+ * Pre-processor Definitions
+ ****************************************************************************/
+
+/* Renumber idle task PIDs
+ *  In NuttX, PID number less than NCPUS are idle tasks.
+ *  In Linux, there is only one idle task of PID 0.
+ */
+
+#define get_pid(pid)  ((pid) < NCPUS ? 0 : (pid))
+
+#define get_task_state(s) ((s) <= LAST_READY_TO_RUN_STATE ? 'R' : 'S')
+
+/****************************************************************************
+ * Private Types
+ ****************************************************************************/
+
+/* The structure to hold the context data of trace dump */
+
+struct trace_dump_cpu_context_s
+{
+  bool ininterrupt;       /* In interrupt handler flag */
+  bool pendingswitch;     /* sched_switch pending flag */
+  int current_state;      /* Task state of the current line */
+  pid_t current_pid;      /* Task PID of the current line */
+  pid_t next_pid;         /* Task PID of the next line */
+};
+
+struct trace_dump_task_context_s
+{
+  FAR struct trace_dump_task_context_s *next;
+  pid_t pid;                              /* Task PID */
+  int syscall_nest;                       /* Syscall nest level */
+  char name[CONFIG_TASK_NAME_SIZE + 1];   /* Task name (with NUL terminator) */
+};
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static struct trace_dump_cpu_context_s g_dump_cpu_ctx[NCPUS];
+static FAR struct trace_dump_task_context_s *g_dump_task_ctx = NULL;
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: trace_dump_init_context
+ ****************************************************************************/
+
+static void trace_dump_init_context(void)
+{
+  int cpu;
+
+  /* Initialize the trace dump context */
+
+  for (cpu = 0; cpu < NCPUS; cpu++)
+    {
+      g_dump_cpu_ctx[cpu].ininterrupt = false;
+      g_dump_cpu_ctx[cpu].pendingswitch = false;
+      g_dump_cpu_ctx[cpu].current_state = TSTATE_TASK_RUNNING;
+      g_dump_cpu_ctx[cpu].current_pid = 0;
+      g_dump_cpu_ctx[cpu].next_pid = 0;
+    }
+}
+
+/****************************************************************************
+ * Name: trace_dump_fini_context
+ ****************************************************************************/
+
+static void trace_dump_fini_context(void)
+{
+  FAR struct trace_dump_task_context_s *tctx;
+  FAR struct trace_dump_task_context_s *ntctx;
+
+  /* Finalize the trace dump context */
+
+  tctx = g_dump_task_ctx;
+  g_dump_task_ctx = NULL;
+  while (tctx != NULL)
+    {
+      ntctx = tctx->next;
+      free(tctx);
+      tctx = ntctx;
+    }
+}
+
+/****************************************************************************
+ * Name: copy_task_name
+ ****************************************************************************/
+
+#if CONFIG_TASK_NAME_SIZE > 0
+static void copy_task_name(FAR char *dst, FAR const char *src)
+{
+  char c;
+  int i;
+
+  /* Replace space to underline
+   * Text trace data format cannot use a space as a task name.
+   */
+
+  for (i = 0; i < CONFIG_TASK_NAME_SIZE; i++)
+    {
+      c = *src++;
+      if (c == '\0')
+        {
+          break;
+        }
+
+      *dst++ = (c == ' ') ? '_' : c;
+    }
+
+  *dst = '\0';
+}
+#endif
+
+/****************************************************************************
+ * Name: get_task_context
+ ****************************************************************************/
+
+FAR static struct trace_dump_task_context_s *get_task_context(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s **tctxp;
+  tctxp = &g_dump_task_ctx;
+  while (*tctxp != NULL)
+    {
+      if ((*tctxp)->pid == pid)
+        {
+          return *tctxp;
+        }
+
+      tctxp = &((*tctxp)->next);
+    }
+
+  /* Create new trace dump task context */
+
+  *tctxp = (FAR struct trace_dump_task_context_s *)
+           malloc(sizeof(struct trace_dump_task_context_s));
+  if (*tctxp != NULL)
+    {
+      (*tctxp)->next = NULL;
+      (*tctxp)->pid = pid;
+      (*tctxp)->syscall_nest = 0;
+      (*tctxp)->name[0] = '\0';
+    }
+
+  return *tctxp;
+}
+
+/****************************************************************************
+ * Name: get_task_name
+ ****************************************************************************/
+
+static const char *get_task_name(pid_t pid)
+{
+  FAR struct trace_dump_task_context_s *tctxp;
+
+  tctxp = get_task_context(pid);
+  if (tctxp != NULL && tctxp->name[0] != '\0')
+    {
+      return tctxp->name;
+    }
+
+  return "<noname>";
+}
+
+/****************************************************************************
+ * Name: trace_dump_header
+ ****************************************************************************/
+
+static void trace_dump_header(FILE *out,
+                              FAR struct note_common_s *note)
+{
+  pid_t pid;
+  uint32_t systime = note->nc_systime[0] +
+                     (note->nc_systime[1] << 8) +
+                     (note->nc_systime[2] << 16) +
+                     (note->nc_systime[3] << 24);
+#ifdef CONFIG_SMP
+  int cpu = note->nc_cpu;
+#else
+  int cpu = 0;
+#endif
+
+  pid = g_dump_cpu_ctx[cpu].current_pid;
+
+  fprintf(out, "%8s-%-3u [%d] %3u.%09u: ",
+          get_task_name(pid), get_pid(pid), cpu,
+          systime / (1000 * 1000 / CONFIG_USEC_PER_TICK),
+          (systime % (1000 * 1000 / CONFIG_USEC_PER_TICK))
+            * CONFIG_USEC_PER_TICK * 1000);

Review comment:
       why need multiple by CONFIG_USEC_PER_TICK?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;

Review comment:
       should we clear flag if all mask is clear? or let user invoke mode command manually?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"
+                                " Output the trace result\n"
+#endif
+          "  mode [{+|-}{o|s|i}...]          :"
+                                " Set task trace options\n"
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          "  syscall [{+|-}<syscallname>...] :"
+                                " Configure syscall trace filter\n"
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          "  irq [{+|-}<irqnum>...]          :"
+                                " Configure IRQ trace filter\n"
+#endif
+         );
+  exit(exitcode);

Review comment:
       should we let caller return/exit to avoid the code dup?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);

Review comment:
       should we restore the old state?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)

Review comment:
       add FAR to all pointer type?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"
+                                " Output the trace result\n"
+#endif
+          "  mode [{+|-}{o|s|i}...]          :"
+                                " Set task trace options\n"
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          "  syscall [{+|-}<syscallname>...] :"
+                                " Configure syscall trace filter\n"
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          "  irq [{+|-}<irqnum>...]          :"
+                                " Configure IRQ trace filter\n"
+#endif
+         );
+  exit(exitcode);
+}
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+int main(int argc, FAR char *argv[])
+{
+  int exitcode;
+  int i;
+
+  exitcode = EXIT_FAILURE;

Review comment:
       merge to line 727?

##########
File path: system/trace/trace.c
##########
@@ -0,0 +1,812 @@
+/****************************************************************************
+ * apps/system/trace/trace.c
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.  The
+ * ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Included Files
+ ****************************************************************************/
+
+#include <nuttx/config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <nuttx/note/notectl_driver.h>
+#ifdef CONFIG_DRIVER_NOTERAM
+#  include <nuttx/note/noteram_driver.h>
+#endif
+
+#include "trace.h"
+
+/****************************************************************************
+ * Private Data
+ ****************************************************************************/
+
+static int g_notectlfd;   /* /dev/notectl file descriptor */
+
+/****************************************************************************
+ * Private Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: notectl_enable
+ ****************************************************************************/
+
+static void notectl_enable(int flag)
+{
+  struct note_filter_mode_s mode;
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(&mode, NULL);
+#else
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#endif
+
+  if (flag)
+    {
+      mode.flag |= NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+  else
+    {
+      mode.flag &= ~NOTE_FILTER_MODE_FLAG_ENABLE;
+    }
+
+#ifdef CONFIG_BUILD_FLAT
+  sched_note_filter_mode(NULL, &mode);
+#else
+  ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#endif
+}
+
+/****************************************************************************
+ * Name: note_ioctl
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static void note_ioctl(int cmd, unsigned long arg)
+{
+  int fd;
+
+  fd = open("/dev/note", O_RDONLY);
+  if (fd < 0)
+    {
+      fprintf(stderr,
+             "trace: cannot open /dev/note\n");
+      return;
+    }
+
+  ioctl(fd, cmd, arg);
+  close(fd);
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_start
+ ****************************************************************************/
+
+static int trace_cmd_start(int index, int argc, char **argv)
+{
+  char *endptr;
+  int duration = 0;
+
+  /* Usage: trace start [<duration>] */
+
+  if (index < argc - 1)
+    {
+      index++;
+      duration = strtoul(argv[index], &endptr, 0);
+      if (!duration || endptr == argv[index] || *endptr != '\0')
+        {
+          fprintf(stderr,
+                  "trace start: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Start tracing */
+
+  notectl_enable(true);
+
+  if (duration > 0)
+    {
+      /* If <duration> is given, stop tracing after specified seconds. */
+
+      sleep(duration);
+      notectl_enable(false);
+    }
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: trace_cmd_dump
+ ****************************************************************************/
+
+#ifdef CONFIG_DRIVER_NOTERAM
+static int trace_cmd_dump(int index, int argc, char **argv)
+{
+  FILE *out = stdout;
+  int ret;
+
+  /* Usage: trace dump [<filename>] */
+
+  /* If <filename> is '-' or not given, trace dump is displayed
+   * to stdout.
+   */
+
+  if (index < argc - 1)
+    {
+      index++;
+      if (strcmp(argv[index], "-") != 0)
+        {
+          /* If <filename> is given, open the file stream for output. */
+
+          out = fopen(argv[index], "w");
+          if (out == NULL)
+            {
+              fprintf(stderr,
+                      "trace dump: cannot open '%s'\n", argv[index]);
+              return ERROR;
+            }
+        }
+    }
+
+  /* Stop the tracing before dump */
+
+  notectl_enable(false);
+
+  /* Dump the trace data */
+
+  ret = trace_dump(out);
+
+  /* If needed, close the file stream for dump. */
+
+  if (out != stdout)
+    {
+      fclose(out);
+    }
+
+  if (ret < 0)
+    {
+      fprintf(stderr,
+              "trace dump: dump failed\n");
+      return ERROR;
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_cmd
+ ****************************************************************************/
+
+#ifdef CONFIG_SYSTEM_SYSTEM
+static int trace_cmd_cmd(int index, int argc, char **argv)
+{
+  char command[CONFIG_NSH_LINELEN];
+
+  /* Usage: trace cmd <command> [<args>...] */
+
+  index++;
+  if (index >= argc)
+    {
+      /* <command> parameter is mandatory. */
+
+      fprintf(stderr,
+              "trace cmd: no argument\n");
+      return ERROR;
+    }
+
+  memset(command, 0, sizeof(command));
+  while (index < argc)
+    {
+      strcat(command, argv[index]);
+      strcat(command, " ");
+      index++;
+    }
+
+  /* Clear the trace buffer */
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_CLEAR, 0);
+#endif
+
+  /* Execute the command with tracing */
+
+  notectl_enable(true);
+  system(command);
+  notectl_enable(false);
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_mode
+ ****************************************************************************/
+
+static int trace_cmd_mode(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+#ifdef CONFIG_DRIVER_NOTERAM
+  unsigned int owmode = 0;
+#endif
+  bool enable;
+  bool modified;
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  struct note_filter_syscall_s filter_syscall;
+#endif
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  struct note_filter_irq_s filter_irq;
+#endif
+  int i;
+  int count;
+
+  /* Usage: trace mode [{+|-}{o|s|i}...] */
+
+  /* Get current trace mode */
+
+  ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+  note_ioctl(NOTERAM_GETMODE, (unsigned long)&owmode);
+#endif
+
+  modified = false;
+
+  /* Parse the mode setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      enable = (argv[index][0] == '+');
+
+      switch (argv[index][1])
+        {
+#ifdef CONFIG_DRIVER_NOTERAM
+          case 'o':   /* Overwrite mode */
+            if (enable)
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_ENABLE;
+              }
+            else
+              {
+                owmode = NOTERAM_MODE_OVERWRITE_DISABLE;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+          case 's':   /* Syscall trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_SYSCALL;
+              }
+            break;
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+          case 'i':   /* IRQ trace */
+            if (enable)
+              {
+                mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            else
+              {
+                mode.flag &= ~NOTE_FILTER_MODE_FLAG_IRQ;
+              }
+            break;
+#endif
+
+          default:
+            fprintf(stderr,
+                    "trace mode: invalid option '%s'\n", argv[index]);
+            return ERROR;
+        }
+
+      modified = true;
+    }
+
+  if (modified)
+    {
+      /* Update trace mode */
+
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+#ifdef CONFIG_DRIVER_NOTERAM
+      note_ioctl(NOTERAM_SETMODE, (unsigned long)&owmode);
+#endif
+
+      return index;
+    }
+
+  /* If no parameter, display current trace mode setting. */
+
+  printf("Task trace mode:\n");
+  printf(" Trace                   : %s\n",
+         (mode.flag & NOTE_FILTER_MODE_FLAG_ENABLE) ?
+          "enabled" : "disabled");
+
+#ifdef CONFIG_DRIVER_NOTERAM
+  printf(" Overwrite               : %s\n",
+         (owmode == NOTERAM_MODE_OVERWRITE_ENABLE) ?
+          "on  (+o)" : "off (-o)");
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+  count = 0;
+  for (i = 0; i < SYS_nsyscalls; i++)
+    {
+      if (NOTE_FILTER_SYSCALLMASK_ISSET(i, &filter_syscall))
+        {
+          count++;
+        }
+    }
+
+  printf(" Syscall trace           : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL ?
+          "on  (+s)" : "off (-s)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_SYSCALL)
+    {
+      printf("  Filtered Syscalls      : %d\n", count);
+    }
+#endif
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+  count = 0;
+  for (i = 0; i < NR_IRQS; i++)
+    {
+      if (NOTE_FILTER_IRQMASK_ISSET(i, &filter_irq))
+        {
+          count++;
+        }
+    }
+
+  printf(" IRQ trace               : %s\n",
+         mode.flag & NOTE_FILTER_MODE_FLAG_IRQ ?
+          "on  (+i)" : "off (-i)");
+  if (mode.flag & NOTE_FILTER_MODE_FLAG_IRQ)
+    {
+      printf("  Filtered IRQs          : %d\n", count);
+    }
+#endif
+
+  return index;
+}
+
+/****************************************************************************
+ * Name: match_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int match_syscall(FAR const char *pattern, FAR const char *name)
+{
+  FAR const char *p = pattern;
+  FAR const char *n = name;
+
+  /* Simple wildcard matcher to specify the syscalls to be masked */
+
+  while (1)
+    {
+      if (*n == '\0')
+        {
+          if (*p == '*')
+            {
+              p++;
+            }
+
+          return *p == '\0';
+        }
+      else if (*p == '\0')
+        {
+          return false;
+        }
+      else if (*p == '*')
+        {
+          if (p[1] == '\0')
+            {
+              return true;
+            }
+          else if (match_syscall(p, n + 1))
+            {
+              return true;
+            }
+          else if (match_syscall(p + 1, n))
+            {
+              return true;
+            }
+
+          return false;
+        }
+      else if (*p == *n)
+        {
+          p++;
+          n++;
+        }
+      else
+        {
+          return false;
+        }
+    }
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_syscall
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_SYSCALL
+static int trace_cmd_syscall(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int syscallno;
+  FAR struct note_filter_syscall_s filter_syscall;
+  int n;
+  int count;
+
+  /* Usage: trace syscall [{+|-}<syscallname>...] */
+
+  /* Get current syscall filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETSYSCALLFILTER,
+        (unsigned long)&filter_syscall);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      /* Check whether the given pattern matches for each syscall names */
+
+      for (syscallno = 0; syscallno < SYS_nsyscalls; syscallno++)
+        {
+          if (!match_syscall(&argv[index][1], g_funcnames[syscallno]))
+            {
+              continue;
+            }
+
+          /* If matches, update the masked syscall number list */
+
+          if (enable)
+            {
+              NOTE_FILTER_SYSCALLMASK_SET(syscallno, &filter_syscall);
+            }
+          else
+            {
+              NOTE_FILTER_SYSCALLMASK_CLR(syscallno, &filter_syscall);
+            }
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current syscall filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETSYSCALLFILTER,
+            (unsigned long)&filter_syscall);
+
+      /* Enable syscall trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_SYSCALL;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered Syscalls: %d\n", count);
+
+      for (n = 0; n < SYS_nsyscalls; n++)
+        {
+          if (NOTE_FILTER_SYSCALLMASK_ISSET(n, &filter_syscall))
+            {
+              printf("  %s\n", g_funcnames[n]);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: trace_cmd_irq
+ ****************************************************************************/
+
+#ifdef CONFIG_SCHED_INSTRUMENTATION_IRQHANDLER
+static int trace_cmd_irq(int index, int argc, char **argv)
+{
+  struct note_filter_mode_s mode;
+  bool enable;
+  bool modified = false;
+  int irqno;
+  char *endptr;
+  struct note_filter_irq_s filter_irq;
+  int n;
+  int count;
+
+  /* Usage: trace irq [{+|-}<irqnum>...] */
+
+  /* Get current irq filter setting */
+
+  ioctl(g_notectlfd, NOTECTL_GETIRQFILTER, (unsigned long)&filter_irq);
+
+  /* Parse the setting parameters */
+
+  while (index < argc - 1)
+    {
+      if (argv[index + 1][0] != '-' && argv[index + 1][0] != '+')
+        {
+          break;
+        }
+
+      index++;
+      modified = true;
+      enable = (argv[index][0] == '+');
+
+      if (argv[index][1] == '*')
+        {
+          /* Mask or unmask all IRQs */
+
+          if (enable)
+            {
+              for (n = 0; n < NR_IRQS; n++)
+                {
+                  NOTE_FILTER_IRQMASK_SET(n, &filter_irq);
+                }
+            }
+          else
+            {
+              NOTE_FILTER_IRQMASK_ZERO(&filter_irq);
+            }
+
+          continue;
+        }
+
+      /* Get IRQ number */
+
+      irqno = strtoul(&argv[index][1], &endptr, 0);
+      if (endptr == &argv[index][1] || *endptr != '\0' ||
+          irqno >= NR_IRQS)
+        {
+          fprintf(stderr,
+                  "trace irq: invalid argument '%s'\n", argv[index]);
+          return ERROR;
+        }
+
+      /* Update the masked IRQ number list */
+
+      if (enable)
+        {
+          NOTE_FILTER_IRQMASK_SET(irqno, &filter_irq);
+        }
+      else
+        {
+          NOTE_FILTER_IRQMASK_CLR(irqno, &filter_irq);
+        }
+    }
+
+  if (modified)
+    {
+      /* Update current irq filter setting */
+
+      ioctl(g_notectlfd, NOTECTL_SETIRQFILTER, (unsigned long)&filter_irq);
+
+      /* Enable irq trace flag */
+
+      ioctl(g_notectlfd, NOTECTL_GETMODE, (unsigned long)&mode);
+      mode.flag |= NOTE_FILTER_MODE_FLAG_IRQ;
+      ioctl(g_notectlfd, NOTECTL_SETMODE, (unsigned long)&mode);
+    }
+  else
+    {
+      /* If no parameter, display current setting. */
+
+      count = 0;
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              count++;
+            }
+        }
+
+      printf("Filtered IRQs: %d\n", count);
+
+      for (n = 0; n < NR_IRQS; n++)
+        {
+          if (NOTE_FILTER_IRQMASK_ISSET(n, &filter_irq))
+            {
+              printf("  %d\n", n);
+            }
+        }
+    }
+
+  return index;
+}
+#endif
+
+/****************************************************************************
+ * Name: show_usage
+ ****************************************************************************/
+
+static void show_usage(FAR const char *progname, int exitcode)
+{
+  fprintf(stderr,
+          "\nUsage: trace <subcommand>...\n"
+          "Subcommand:\n"
+          "  start [<duration>]              :"
+                                " Start task tracing\n"
+          "  stop                            :"
+                                " Stop task tracing\n"
+#ifdef CONFIG_SYSTEM_SYSTEM
+          "  cmd <command> [<args>...]       :"
+                                " Get the trace while running <command>\n"
+#endif
+#ifdef CONFIG_DRIVER_NOTERAM
+          "  dump [<filename>]               :"

Review comment:
       Since trace without argument is same as trace dump wihtout filename, how about we move {+|-}o to here to centralize all dump option in one place? e.g.:
   dump ->show the overwrite state
   dump filename->save to file
   dump +o->overwrite
   dump -o->don't ovwerwrite
   The benefit is that, we can add function to trace_dump.c like:
   trace_dump_cmd
   trace_dump_show_usage
   and trace.c just forward to these function.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org