(nuttx) 02/04: fs/inode: Enforce RLIMIT_NOFILE for new file descriptors.

[ac...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

acassis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nuttx.git

commit a6d583afb4118e504bead6f9a390f24ce1899b2f
Author: Marco Casaroli <ma...@gmail.com>
AuthorDate: Sun Oct 18 20:06:19 2026 +0200

    fs/inode: Enforce RLIMIT_NOFILE for new file descriptors.
    
    A process could open files up to OPEN_MAX, whatever its RLIMIT_NOFILE
    soft limit was.
    
    A new file descriptor must now be below the soft limit of the process,
    as POSIX says: open(), dup() and fcntl(F_DUPFD) fail with EMFILE when
    the lowest free descriptor is at or above it (F_DUPFD with EINVAL when
    the lowest descriptor that it may use is already at or above it), and
    dup2() and dup3() fail with EBADF for a target at or above it.
    
    Assisted-by: Claude Code:claude-opus-5-5
    Signed-off-by: Marco Casaroli <ma...@gmail.com>
---
 fs/inode/fs_files.c | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/fs/inode/fs_files.c b/fs/inode/fs_files.c
index 6407f0eb9bd..3dc715ce31e 100644
--- a/fs/inode/fs_files.c
+++ b/fs/inode/fs_files.c
@@ -26,6 +26,7 @@
 
 #include <nuttx/config.h>
 
+#include <sys/param.h>
 #include <sys/types.h>
 #include <string.h>
 #include <assert.h>
@@ -45,6 +46,7 @@
 #include <nuttx/spawn.h>
 #include <nuttx/spinlock.h>
 #include <nuttx/lib/lib.h>
+#include <nuttx/nuttx.h>
 
 #ifdef CONFIG_FDCHECK
 #  include <nuttx/fdcheck.h>
@@ -58,6 +60,23 @@
  * Private Functions
  ****************************************************************************/
 
+/****************************************************************************
+ * Name: fdlist_limit
+ *
+ * Description:
+ *   Return the RLIMIT_NOFILE soft limit of the task group that owns the
+ *   list: a new file descriptor must be below it.
+ *
+ ****************************************************************************/
+
+static inline int fdlist_limit(FAR struct fdlist *list)
+{
+  FAR struct task_group_s *group =
+    container_of(list, struct task_group_s, tg_fdlist);
+
+  return MIN(group->tg_rlimit_nofile.rlim_cur, OPEN_MAX);
+}
+
 /****************************************************************************
  * Name: fdlist_get_by_index
  ****************************************************************************/
@@ -319,7 +338,7 @@ int fdlist_dup3(FAR struct fdlist *list, int fd1, int fd2, int flags)
 
   /* Get the file descriptor list.  It should not be NULL in this context. */
 
-  if (fd2 < 0)
+  if (fd2 < 0 || fd2 >= fdlist_limit(list))
     {
       return -EBADF;
     }
@@ -568,6 +587,7 @@ int fdlist_dupfile(FAR struct fdlist *list, int oflags, int minfd,
 {
   FAR struct fd *fdp;
   irqstate_t flags;
+  int limit;
   int ret;
   int i;
   int j;
@@ -578,6 +598,12 @@ int fdlist_dupfile(FAR struct fdlist *list, int oflags, int minfd,
   minfd = fdcheck_restore(minfd);
 #endif
 
+  limit = fdlist_limit(list);
+  if (minfd >= limit)
+    {
+      return -EINVAL;
+    }
+
   i = minfd / CONFIG_NFILE_DESCRIPTORS_PER_BLOCK;
   j = minfd % CONFIG_NFILE_DESCRIPTORS_PER_BLOCK;
 
@@ -605,6 +631,14 @@ int fdlist_dupfile(FAR struct fdlist *list, int oflags, int minfd,
           fdp = &list->fl_fds[i][j];
           if (fdp->f_file == NULL)
             {
+              /* The lowest free descriptor is beyond RLIMIT_NOFILE */
+
+              if (i * CONFIG_NFILE_DESCRIPTORS_PER_BLOCK + j >= limit)
+                {
+                  spin_unlock_irqrestore_notrace(&list->fl_lock, flags);
+                  return -EMFILE;
+                }
+
               atomic_add(&filep->f_refs, 1);
               fdp->f_file        = filep;
               fdp->f_cloexec     = !!(oflags & O_CLOEXEC);