You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apr.apache.org by iv...@apache.org on 2019/06/07 11:02:55 UTC

svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Author: ivan
Date: Fri Jun  7 11:02:55 2019
New Revision: 1860745

URL: http://svn.apache.org/viewvc?rev=1860745&view=rev
Log:
Windows platform: Remove ANSI codepath from apr_dir_*().

Modified:
    apr/apr/trunk/file_io/win32/dir.c

Modified: apr/apr/trunk/file_io/win32/dir.c
URL: http://svn.apache.org/viewvc/apr/apr/trunk/file_io/win32/dir.c?rev=1860745&r1=1860744&r2=1860745&view=diff
==============================================================================
--- apr/apr/trunk/file_io/win32/dir.c (original)
+++ apr/apr/trunk/file_io/win32/dir.c Fri Jun  7 11:02:55 2019
@@ -63,31 +63,10 @@ APR_DECLARE(apr_status_t) apr_dir_open(a
     (*new)->dirname[len++] = '\0';
     (*new)->dirname[len] = '\0';
 
-#if APR_HAS_UNICODE_FS
-    IF_WIN_OS_IS_UNICODE
-    {
-        /* Create a buffer for the longest file name we will ever see 
-         */
-        (*new)->w.entry = apr_pcalloc(pool, sizeof(WIN32_FIND_DATAW));
-        (*new)->name = apr_pcalloc(pool, APR_FILE_MAX * 3 + 1);        
-    }
-#endif
-#if APR_HAS_ANSI_FS
-    ELSE_WIN_OS_IS_ANSI
-    {
-        /* Note that we won't open a directory that is greater than MAX_PATH,
-         * counting the additional '/' '*' wildcard suffix.  If a * won't fit
-         * then neither will any other file name within the directory.
-         * The length not including the trailing '*' is stored as rootlen, to
-         * skip over all paths which are too long.
-         */
-        if (len >= APR_PATH_MAX) {
-            (*new) = NULL;
-            return APR_ENAMETOOLONG;
-        }
-        (*new)->n.entry = apr_pcalloc(pool, sizeof(WIN32_FIND_DATAW));
-    }
-#endif
+    /* Create a buffer for the longest file name we will ever see 
+        */
+    (*new)->w.entry = apr_pcalloc(pool, sizeof(WIN32_FIND_DATAW));
+    (*new)->name = apr_pcalloc(pool, APR_FILE_MAX * 3 + 1);
     (*new)->rootlen = len - 1;
     (*new)->pool = pool;
     (*new)->dirhand = INVALID_HANDLE_VALUE;
@@ -114,116 +93,73 @@ APR_DECLARE(apr_status_t) apr_dir_read(a
 {
     apr_status_t rv;
     char *fname;
+    apr_wchar_t wdirname[APR_PATH_MAX];
+    apr_wchar_t* eos = NULL;
+
     /* The while loops below allow us to skip all invalid file names, so that
      * we aren't reporting any files where their absolute paths are too long.
      */
-#if APR_HAS_UNICODE_FS
-    apr_wchar_t wdirname[APR_PATH_MAX];
-    apr_wchar_t *eos = NULL;
-    IF_WIN_OS_IS_UNICODE
-    {
-        /* This code path is always be invoked by apr_dir_open or
-         * apr_dir_rewind, so return without filling out the finfo.
-         */
-        if (thedir->dirhand == INVALID_HANDLE_VALUE) 
-        {
-            apr_status_t rv;
-            FINDEX_INFO_LEVELS info_level;
-
-            if ((rv = utf8_to_unicode_path(wdirname, sizeof(wdirname) 
-                                                   / sizeof(apr_wchar_t), 
-                                           thedir->dirname))) {
-                return rv;
-            }
-            eos = wcschr(wdirname, '\0');
-            eos[0] = '*';
-            eos[1] = '\0';
 
-            /* Do not request short file names on Windows 7 and later. */
-            if (apr_os_level >= APR_WIN_7) {
-                info_level = FindExInfoBasic;
-            }
-            else {
-                info_level = FindExInfoStandard;
-            }
+    /* This code path is always be invoked by apr_dir_open or
+     * apr_dir_rewind, so return without filling out the finfo.
+     */
+    if (thedir->dirhand == INVALID_HANDLE_VALUE) 
+    {
+        apr_status_t rv;
+        FINDEX_INFO_LEVELS info_level;
 
-            thedir->dirhand = FindFirstFileExW(wdirname, info_level,
-                                               thedir->w.entry,
-                                               FindExSearchNameMatch, NULL,
-                                               0);
-            eos[0] = '\0';
-            if (thedir->dirhand == INVALID_HANDLE_VALUE) {
-                return apr_get_os_error();
-            }
-            thedir->bof = 1;
-            return APR_SUCCESS;
-        }
-        else if (thedir->bof) {
-            /* Noop - we already called FindFirstFileW from
-             * either apr_dir_open or apr_dir_rewind ... use
-             * that first record.
-             */
-            thedir->bof = 0; 
+        if ((rv = utf8_to_unicode_path(wdirname, sizeof(wdirname) 
+                                                / sizeof(apr_wchar_t), 
+                                        thedir->dirname))) {
+            return rv;
         }
-        else if (!FindNextFileW(thedir->dirhand, thedir->w.entry)) {
+        eos = wcschr(wdirname, '\0');
+        eos[0] = '*';
+        eos[1] = '\0';
+
+        /* Do not request short file names on Windows 7 and later. */
+        if (apr_os_level >= APR_WIN_7) {
+            info_level = FindExInfoBasic;
+        }
+        else {
+            info_level = FindExInfoStandard;
+        }
+
+        thedir->dirhand = FindFirstFileExW(wdirname, info_level,
+                                           thedir->w.entry,
+                                           FindExSearchNameMatch, NULL,
+                                           0);
+        eos[0] = '\0';
+        if (thedir->dirhand == INVALID_HANDLE_VALUE) {
             return apr_get_os_error();
         }
-
-        while (thedir->rootlen &&
-               thedir->rootlen + wcslen(thedir->w.entry->cFileName) >= APR_PATH_MAX)
-        {
-            if (!FindNextFileW(thedir->dirhand, thedir->w.entry)) {
-                return apr_get_os_error();
-            }
-        }
-        if ((rv = unicode_to_utf8_path(thedir->name, APR_FILE_MAX * 3 + 1, 
-                                       thedir->w.entry->cFileName)))
-            return rv;
-        fname = thedir->name;
+        thedir->bof = 1;
+        return APR_SUCCESS;
     }
-#endif
-#if APR_HAS_ANSI_FS
-    ELSE_WIN_OS_IS_ANSI
-    {
-        /* This code path is always be invoked by apr_dir_open or 
-         * apr_dir_rewind, so return without filling out the finfo.
+    else if (thedir->bof) {
+        /* Noop - we already called FindFirstFileW from
+         * either apr_dir_open or apr_dir_rewind ... use
+         * that first record.
          */
-        if (thedir->dirhand == INVALID_HANDLE_VALUE) {
-            /* '/' terminated, so add the '*' and pop it when we finish */
-            char *eop = strchr(thedir->dirname, '\0');
-            eop[0] = '*';
-            eop[1] = '\0';
-            thedir->dirhand = FindFirstFileA(thedir->dirname, 
-                                             thedir->n.entry);
-            eop[0] = '\0';
-            if (thedir->dirhand == INVALID_HANDLE_VALUE) {
-                return apr_get_os_error();
-            }
-            thedir->bof = 1;
-            return APR_SUCCESS;
-        }
-        else if (thedir->bof) {
-            /* Noop - we already called FindFirstFileW from
-             * either apr_dir_open or apr_dir_rewind ... use
-             * that first record.
-             */
-            thedir->bof = 0; 
-        }
-        else if (!FindNextFileA(thedir->dirhand, thedir->n.entry)) {
+        thedir->bof = 0; 
+    }
+    else if (!FindNextFileW(thedir->dirhand, thedir->w.entry)) {
+        return apr_get_os_error();
+    }
+
+    while (thedir->rootlen &&
+           thedir->rootlen + wcslen(thedir->w.entry->cFileName) >= APR_PATH_MAX)
+    {
+        if (!FindNextFileW(thedir->dirhand, thedir->w.entry)) {
             return apr_get_os_error();
         }
-        while (thedir->rootlen &&
-               thedir->rootlen + strlen(thedir->n.entry->cFileName) >= MAX_PATH)
-        {
-            if (!FindNextFileA(thedir->dirhand, thedir->n.entry)) {
-                return apr_get_os_error();
-            }
-        }
-        fname = thedir->n.entry->cFileName;
     }
-#endif
+    if ((rv = unicode_to_utf8_path(thedir->name, APR_FILE_MAX * 3 + 1, 
+                                   thedir->w.entry->cFileName)))
+        return rv;
+    fname = thedir->name;
 
-    fillin_fileinfo(finfo, (WIN32_FILE_ATTRIBUTE_DATA *) thedir->w.entry, 
+    fillin_fileinfo(finfo, (WIN32_FILE_ATTRIBUTE_DATA *) thedir->w.entry,
                     0, 1, fname, wanted);
     finfo->pool = thedir->pool;
 
@@ -233,39 +169,15 @@ APR_DECLARE(apr_status_t) apr_dir_read(a
     if (wanted &= ~finfo->valid) {
         /* Go back and get more_info if we can't answer the whole inquiry
          */
-#if APR_HAS_UNICODE_FS
-        IF_WIN_OS_IS_UNICODE
-        {
-            /* Almost all our work is done.  Tack on the wide file name
-             * to the end of the wdirname (already / delimited)
-             */
-            if (!eos)
-                eos = wcschr(wdirname, '\0');
-            wcscpy(eos, thedir->w.entry->cFileName);
-            rv = more_finfo(finfo, wdirname, wanted, MORE_OF_WFSPEC);
-            eos[0] = '\0';
-            return rv;
-        }
-#endif
-#if APR_HAS_ANSI_FS
-        ELSE_WIN_OS_IS_ANSI
-        {
-#if APR_HAS_UNICODE_FS
-            /* Don't waste stack space on a second buffer, the one we set
-             * aside for the wide directory name is twice what we need.
-             */
-            char *fspec = (char*)wdirname;
-#else
-            char fspec[APR_PATH_MAX];
-#endif
-            apr_size_t dirlen = strlen(thedir->dirname);
-            if (dirlen >= sizeof(fspec))
-                dirlen = sizeof(fspec) - 1;
-            apr_cpystrn(fspec, thedir->dirname, sizeof(fspec));
-            apr_cpystrn(fspec + dirlen, fname, sizeof(fspec) - dirlen);
-            return more_finfo(finfo, fspec, wanted, MORE_OF_FSPEC);
-        }
-#endif
+        /* Almost all our work is done.  Tack on the wide file name
+         * to the end of the wdirname (already / delimited)
+         */
+        if (!eos)
+            eos = wcschr(wdirname, '\0');
+        wcscpy(eos, thedir->w.entry->cFileName);
+        rv = more_finfo(finfo, wdirname, wanted, MORE_OF_WFSPEC);
+        eos[0] = '\0';
+        return rv;
     }
 
     return APR_SUCCESS;
@@ -289,27 +201,17 @@ APR_DECLARE(apr_status_t) apr_dir_rewind
 APR_DECLARE(apr_status_t) apr_dir_make(const char *path, apr_fileperms_t perm,
                                        apr_pool_t *pool)
 {
-#if APR_HAS_UNICODE_FS
-    IF_WIN_OS_IS_UNICODE
-    {
-        apr_wchar_t wpath[APR_PATH_MAX];
-        apr_status_t rv;
-        if ((rv = utf8_to_unicode_path(wpath,
-                                       sizeof(wpath) / sizeof(apr_wchar_t),
-                                       path))) {
-            return rv;
-        }
-        if (!CreateDirectoryW(wpath, NULL)) {
-            return apr_get_os_error();
-        }
+    apr_wchar_t wpath[APR_PATH_MAX];
+    apr_status_t rv;
+    if ((rv = utf8_to_unicode_path(wpath,
+                                    sizeof(wpath) / sizeof(apr_wchar_t),
+                                    path))) {
+        return rv;
     }
-#endif
-#if APR_HAS_ANSI_FS
-    ELSE_WIN_OS_IS_ANSI
-        if (!CreateDirectory(path, NULL)) {
-            return apr_get_os_error();
-        }
-#endif
+    if (!CreateDirectoryW(wpath, NULL)) {
+        return apr_get_os_error();
+    }
+
     return APR_SUCCESS;
 }
 
@@ -380,27 +282,18 @@ APR_DECLARE(apr_status_t) apr_dir_make_r
 
 APR_DECLARE(apr_status_t) apr_dir_remove(const char *path, apr_pool_t *pool)
 {
-#if APR_HAS_UNICODE_FS
-    IF_WIN_OS_IS_UNICODE
-    {
-        apr_wchar_t wpath[APR_PATH_MAX];
-        apr_status_t rv;
-        if ((rv = utf8_to_unicode_path(wpath,
-                                       sizeof(wpath) / sizeof(apr_wchar_t),
-                                       path))) {
-            return rv;
-        }
-        if (!RemoveDirectoryW(wpath)) {
-            return apr_get_os_error();
-        }
+    apr_wchar_t wpath[APR_PATH_MAX];
+    apr_status_t rv;
+
+    if ((rv = utf8_to_unicode_path(wpath,
+                                    sizeof(wpath) / sizeof(apr_wchar_t),
+                                    path))) {
+        return rv;
     }
-#endif
-#if APR_HAS_ANSI_FS
-    ELSE_WIN_OS_IS_ANSI
-        if (!RemoveDirectory(path)) {
-            return apr_get_os_error();
-        }
-#endif
+    if (!RemoveDirectoryW(wpath)) {
+        return apr_get_os_error();
+    }
+
     return APR_SUCCESS;
 }
 



Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by William A Rowe Jr <wr...@rowe-clan.net>.
On Tue, Jun 11, 2019 at 9:14 AM William A Rowe Jr <wr...@rowe-clan.net>
wrote:

> On Tue, Jun 11, 2019 at 4:15 AM Branko Čibej <br...@apache.org> wrote:
>
>> On 07.06.2019 21:58, William A Rowe Jr wrote:
>> > I think the optimal way is to allocate a pair of apr thread-specific
>> > wchar buffers in each thread's pool on startup, and use those
>> > exclusively per-thread for wchar translations. We could be looking at
>> > 64k/thread exclusively for name translation, but it doesn't seem
>> > unreasonable.
>> >
>> > The alternative is to continue to use stack, we surely don't want to
>> > lock on acquiring or allocating name translation buffers. /shrug
>>
>> Since this is Windows, and there's no embedded Windows environment left
>> that I'm aware of that's still alive, we can continue with using the
>> stack ... but wouldn't it be so much better to alloca() the required
>> size instead of blindly burning through 64k every time? Obviously that
>> means counting the characters first.
>>
>
> I don't think there is any need to do so. A name buffer needs 32k (right
> now
> we limit that to 8k, but if we wanted to satisfy the OS-acceptable pathname
> length, we would blow that out 4x the current arbitrary limit.) For a
> rename,
> make that 2x buffers. But there is no benefit to wasting cycles determining
> the string length ahead of allocation, because the very next call can run
> right past that limit and hit the wall on available stack. Demanding that
> there
> always be a potential 32k runway of available stack doesn't seem excessive.
>
> The point to the stack is that it contracts immediately on return. So we
> aren't
> burning through a 64k buffer - we are ensuring that we have been topped-up
> to 64k remaining. The targets of these calls are all Win32 API
> invocations,
> we are never nesting them inside further big-buffer allocations of our own.
> What might happen in ntdll we have little control over.
>
>  We either reserve about 2x buffers for file name transliteration in heap
> per thread, or we use the thread stack. As long as we trust that our utf-8
> to ucs-2 logic is rock solid and the allocations and limits are correctly
> coded, this continues to be a safe approach. The 32k/64k are immediately
> given back in the current stack-based approach, but would simply be boat
> anchors most of the time if they are set aside in heap.
>
> I'm +/-0 on switching from stack to heap for this particular transformation
> but welcome good suggestions.
>

I failed to call this out, but guessing that we agree that if we preserve
wide
character string results (without discarding them after a single API call),
using the string size or moving towards counted byte string representation
makes a whole lot more sense for longer-term heap allocations.

Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by William A Rowe Jr <wr...@rowe-clan.net>.
Brane, if that's sufficient, would you propose a rewording patch to dispel
any confusion about how the API behaves?

I understand where you confusion arose, but UCS-2<>UTF-16 was a thing
already when I wrote the code. What I didn't reflect in comments is that we
will throw away bad UTF-16 as quickly as we dispel UTF-8, even as Java
kept eating it up for some very serious consequences (yes, I was the first
reporter, so far as I know.)



On Wed, Jun 12, 2019 at 8:15 PM Branko Čibej <br...@apache.org> wrote:

> On 12.06.2019 16:47, William A Rowe Jr wrote:
> > On Tue, Jun 11, 2019 at 5:38 PM William A Rowe Jr <wrowe@rowe-clan.net
> > <ma...@rowe-clan.net>> wrote:
> >
> >     On Tue, Jun 11, 2019 at 1:44 PM Branko Čibej <brane@apache.org
> >     <ma...@apache.org>> wrote:
> >
> >>          We either reserve about 2x buffers for file name
> >>         transliteration in heap
> >>         per thread, or we use the thread stack. As long as we trust
> >>         that our utf-8
> >>         to ucs-2 logic is rock solid and the allocations and limits
> >>         are correctly
> >>         coded, this continues to be a safe approach.
> >
> >         Apropos of that, for 2.0 we're about to or have already
> >         ditched support for versions of Windows that do not have
> >         native UTF-8/UTF-16 conversions (ah, yes ... Windows has
> >         finally moved from UCS-2 to UTF-16). Wouldn't this be the
> >         right time to switch to using Windows' functions instead of
> >         staying with our own? Especially since, with the transition to
> >         UTF-16, we have to deal correctly with surrogate pairs,
> >         something our current code (IIRC) doesn't do.
> >
> >
> >     A bit of a misnomer, the code is full of references to ucs-2
> >     w/surrogate pair
> >     support, the combo of these is utf-16. The comments can be
> >     refreshed to
> >     today's utf-16 nomenclature.
> >
> >
> > How strongly do we feel about the naming? I have the following patch to
> > commit if we want to observe current convention in utf-naming and
> > deprecate
> > many (but not all) ucs references.
>
> As far as I'm concerned, the naming (which is an internal matter, not
> part of the API) can stay as long as the implementation is correct. A
> comment to the tune of "it's actually UTF-16 but started life in the
> olden days when only UCS-2 existed" is good enough.
>
> -- Brane
>
>

Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by Branko Čibej <br...@apache.org>.
On 12.06.2019 16:47, William A Rowe Jr wrote:
> On Tue, Jun 11, 2019 at 5:38 PM William A Rowe Jr <wrowe@rowe-clan.net
> <ma...@rowe-clan.net>> wrote:
>
>     On Tue, Jun 11, 2019 at 1:44 PM Branko Čibej <brane@apache.org
>     <ma...@apache.org>> wrote:
>
>>          We either reserve about 2x buffers for file name
>>         transliteration in heap
>>         per thread, or we use the thread stack. As long as we trust
>>         that our utf-8
>>         to ucs-2 logic is rock solid and the allocations and limits
>>         are correctly
>>         coded, this continues to be a safe approach.
>
>         Apropos of that, for 2.0 we're about to or have already
>         ditched support for versions of Windows that do not have
>         native UTF-8/UTF-16 conversions (ah, yes ... Windows has
>         finally moved from UCS-2 to UTF-16). Wouldn't this be the
>         right time to switch to using Windows' functions instead of
>         staying with our own? Especially since, with the transition to
>         UTF-16, we have to deal correctly with surrogate pairs,
>         something our current code (IIRC) doesn't do.
>
>
>     A bit of a misnomer, the code is full of references to ucs-2
>     w/surrogate pair
>     support, the combo of these is utf-16. The comments can be
>     refreshed to 
>     today's utf-16 nomenclature.
>
>
> How strongly do we feel about the naming? I have the following patch to
> commit if we want to observe current convention in utf-naming and
> deprecate
> many (but not all) ucs references.

As far as I'm concerned, the naming (which is an internal matter, not
part of the API) can stay as long as the implementation is correct. A
comment to the tune of "it's actually UTF-16 but started life in the
olden days when only UCS-2 existed" is good enough.

-- Brane


Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by William A Rowe Jr <wr...@rowe-clan.net>.
On Tue, Jun 11, 2019 at 5:38 PM William A Rowe Jr <wr...@rowe-clan.net>
wrote:

> On Tue, Jun 11, 2019 at 1:44 PM Branko Čibej <br...@apache.org> wrote:
>
>>  We either reserve about 2x buffers for file name transliteration in heap
>> per thread, or we use the thread stack. As long as we trust that our utf-8
>> to ucs-2 logic is rock solid and the allocations and limits are correctly
>> coded, this continues to be a safe approach.
>>
>>
>> Apropos of that, for 2.0 we're about to or have already ditched support
>> for versions of Windows that do not have native UTF-8/UTF-16 conversions
>> (ah, yes ... Windows has finally moved from UCS-2 to UTF-16). Wouldn't this
>> be the right time to switch to using Windows' functions instead of staying
>> with our own? Especially since, with the transition to UTF-16, we have to
>> deal correctly with surrogate pairs, something our current code (IIRC)
>> doesn't do.
>>
>
> A bit of a misnomer, the code is full of references to ucs-2 w/surrogate
> pair
> support, the combo of these is utf-16. The comments can be refreshed to
> today's utf-16 nomenclature.
>

How strongly do we feel about the naming? I have the following patch to
commit if we want to observe current convention in utf-naming and deprecate
many (but not all) ucs references.

Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by William A Rowe Jr <wr...@rowe-clan.net>.
On Tue, Jun 11, 2019 at 1:44 PM Branko Čibej <br...@apache.org> wrote:

>  We either reserve about 2x buffers for file name transliteration in heap
> per thread, or we use the thread stack. As long as we trust that our utf-8
> to ucs-2 logic is rock solid and the allocations and limits are correctly
> coded, this continues to be a safe approach.
>
>
> Apropos of that, for 2.0 we're about to or have already ditched support
> for versions of Windows that do not have native UTF-8/UTF-16 conversions
> (ah, yes ... Windows has finally moved from UCS-2 to UTF-16). Wouldn't this
> be the right time to switch to using Windows' functions instead of staying
> with our own? Especially since, with the transition to UTF-16, we have to
> deal correctly with surrogate pairs, something our current code (IIRC)
> doesn't do.
>

A bit of a misnomer, the code is full of references to ucs-2 w/surrogate
pair
support, the combo of these is utf-16. The comments can be refreshed to
today's utf-16 nomenclature.

Today's logic remains correct, and of course does the correct thing,
because
an unpaired utf-8 surrogate value would be very broken and even possibly a
security issue, much as decoding other invalid utf-8 bytestreams proved to
be.

If you want to look at win32 api's, feel free to benchmark; though I doubt
it
would outperform the current implementation.

Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by Branko Čibej <br...@apache.org>.
On 11.06.2019 16:14, William A Rowe Jr wrote:
> On Tue, Jun 11, 2019 at 4:15 AM Branko Čibej <brane@apache.org
> <ma...@apache.org>> wrote:
>
>     On 07.06.2019 21:58, William A Rowe Jr wrote:
>     > I think the optimal way is to allocate a pair of apr thread-specific
>     > wchar buffers in each thread's pool on startup, and use those
>     > exclusively per-thread for wchar translations. We could be
>     looking at
>     > 64k/thread exclusively for name translation, but it doesn't seem
>     > unreasonable.
>     >
>     > The alternative is to continue to use stack, we surely don't want to
>     > lock on acquiring or allocating name translation buffers. /shrug
>
>     Since this is Windows, and there's no embedded Windows environment
>     left
>     that I'm aware of that's still alive, we can continue with using the
>     stack ... but wouldn't it be so much better to alloca() the required
>     size instead of blindly burning through 64k every time? Obviously that
>     means counting the characters first.
>
>
> I don't think there is any need to do so. A name buffer needs 32k
> (right now
> we limit that to 8k, but if we wanted to satisfy the OS-acceptable
> pathname
> length, we would blow that out 4x the current arbitrary limit.) For a
> rename,
> make that 2x buffers. But there is no benefit to wasting cycles
> determining
> the string length ahead of allocation, because the very next call can run
> right past that limit and hit the wall on available stack. Demanding
> that there
> always be a potential 32k runway of available stack doesn't seem
> excessive.
>
> The point to the stack is that it contracts immediately on return. So
> we aren't
> burning through a 64k buffer - we are ensuring that we have been topped-up
> to 64k remaining. The targets of these calls are all Win32 API
> invocations, 
> we are never nesting them inside further big-buffer allocations of our
> own.
> What might happen in ntdll we have little control over.

Point. Agreed.

>  We either reserve about 2x buffers for file name transliteration in heap
> per thread, or we use the thread stack. As long as we trust that our utf-8
> to ucs-2 logic is rock solid and the allocations and limits are correctly
> coded, this continues to be a safe approach.

Apropos of that, for 2.0 we're about to or have already ditched support
for versions of Windows that do not have native UTF-8/UTF-16 conversions
(ah, yes ... Windows has finally moved from UCS-2 to UTF-16). Wouldn't
this be the right time to switch to using Windows' functions instead of
staying with our own? Especially since, with the transition to UTF-16,
we have to deal correctly with surrogate pairs, something our current
code (IIRC) doesn't do.

-- Brane


Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by Ivan Zhakov <iv...@visualsvn.com>.
On Tue, 11 Jun 2019 at 17:14, William A Rowe Jr <wr...@rowe-clan.net> wrote:
>
> On Tue, Jun 11, 2019 at 4:15 AM Branko Čibej <br...@apache.org> wrote:
>>
>> On 07.06.2019 21:58, William A Rowe Jr wrote:
>> > I think the optimal way is to allocate a pair of apr thread-specific
>> > wchar buffers in each thread's pool on startup, and use those
>> > exclusively per-thread for wchar translations. We could be looking at
>> > 64k/thread exclusively for name translation, but it doesn't seem
>> > unreasonable.
>> >
>> > The alternative is to continue to use stack, we surely don't want to
>> > lock on acquiring or allocating name translation buffers. /shrug
>>
>> Since this is Windows, and there's no embedded Windows environment left
>> that I'm aware of that's still alive, we can continue with using the
>> stack ... but wouldn't it be so much better to alloca() the required
>> size instead of blindly burning through 64k every time? Obviously that
>> means counting the characters first.
>
>
> I don't think there is any need to do so. A name buffer needs 32k (right now
> we limit that to 8k, but if we wanted to satisfy the OS-acceptable pathname
> length, we would blow that out 4x the current arbitrary limit.) For a rename,
> make that 2x buffers. But there is no benefit to wasting cycles determining
> the string length ahead of allocation, because the very next call can run
> right past that limit and hit the wall on available stack. Demanding that there
> always be a potential 32k runway of available stack doesn't seem excessive.
>
> The point to the stack is that it contracts immediately on return. So we aren't
> burning through a 64k buffer - we are ensuring that we have been topped-up
> to 64k remaining. The targets of these calls are all Win32 API invocations,
> we are never nesting them inside further big-buffer allocations of our own.
> What might happen in ntdll we have little control over.
>
>  We either reserve about 2x buffers for file name transliteration in heap
> per thread, or we use the thread stack. As long as we trust that our utf-8
> to ucs-2 logic is rock solid and the allocations and limits are correctly
> coded, this continues to be a safe approach. The 32k/64k are immediately
> given back in the current stack-based approach, but would simply be boat
> anchors most of the time if they are set aside in heap.
>
> I'm +/-0 on switching from stack to heap for this particular transformation
> but welcome good suggestions.
> .
I agree that in general allocating 16kb on stack should be avoided. However,
I also think that is it's not too problematic in this specific case, as it
happens within a function that only calls the Win32 API.

Also, this particular code should probably be changed anyway to avoid doing
UTF8->UCS conversions per every apr_dir_read(), and I have that on my TODO
list.

Speaking of this commit, my aim here was to fix accessing the uninitialized
memory, as I did in r1860747. And this turned out to be much simpler with
the ANSI code path removed.

--
Ivan Zhakov

Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by William A Rowe Jr <wr...@rowe-clan.net>.
On Tue, Jun 11, 2019 at 4:15 AM Branko Čibej <br...@apache.org> wrote:

> On 07.06.2019 21:58, William A Rowe Jr wrote:
> > I think the optimal way is to allocate a pair of apr thread-specific
> > wchar buffers in each thread's pool on startup, and use those
> > exclusively per-thread for wchar translations. We could be looking at
> > 64k/thread exclusively for name translation, but it doesn't seem
> > unreasonable.
> >
> > The alternative is to continue to use stack, we surely don't want to
> > lock on acquiring or allocating name translation buffers. /shrug
>
> Since this is Windows, and there's no embedded Windows environment left
> that I'm aware of that's still alive, we can continue with using the
> stack ... but wouldn't it be so much better to alloca() the required
> size instead of blindly burning through 64k every time? Obviously that
> means counting the characters first.
>

I don't think there is any need to do so. A name buffer needs 32k (right now
we limit that to 8k, but if we wanted to satisfy the OS-acceptable pathname
length, we would blow that out 4x the current arbitrary limit.) For a
rename,
make that 2x buffers. But there is no benefit to wasting cycles determining
the string length ahead of allocation, because the very next call can run
right past that limit and hit the wall on available stack. Demanding that
there
always be a potential 32k runway of available stack doesn't seem excessive.

The point to the stack is that it contracts immediately on return. So we
aren't
burning through a 64k buffer - we are ensuring that we have been topped-up
to 64k remaining. The targets of these calls are all Win32 API invocations,
we are never nesting them inside further big-buffer allocations of our own.
What might happen in ntdll we have little control over.

 We either reserve about 2x buffers for file name transliteration in heap
per thread, or we use the thread stack. As long as we trust that our utf-8
to ucs-2 logic is rock solid and the allocations and limits are correctly
coded, this continues to be a safe approach. The 32k/64k are immediately
given back in the current stack-based approach, but would simply be boat
anchors most of the time if they are set aside in heap.

I'm +/-0 on switching from stack to heap for this particular transformation
but welcome good suggestions.
.

Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by Branko Čibej <br...@apache.org>.
On 07.06.2019 21:58, William A Rowe Jr wrote:
> I think the optimal way is to allocate a pair of apr thread-specific
> wchar buffers in each thread's pool on startup, and use those
> exclusively per-thread for wchar translations. We could be looking at
> 64k/thread exclusively for name translation, but it doesn't seem
> unreasonable.
>
> The alternative is to continue to use stack, we surely don't want to
> lock on acquiring or allocating name translation buffers. /shrug

Since this is Windows, and there's no embedded Windows environment left
that I'm aware of that's still alive, we can continue with using the
stack ... but wouldn't it be so much better to alloca() the required
size instead of blindly burning through 64k every time? Obviously that
means counting the characters first.


> On Fri, Jun 7, 2019 at 6:27 AM Greg Stein <gstein@gmail.com
> <ma...@gmail.com>> wrote:
>
>     On Fri, Jun 7, 2019 at 6:02 AM <ivan@apache.org
>     <ma...@apache.org>> wrote:
>     >...
>
>         \@@ -114,116 +93,73 @@ APR_DECLARE(apr_status_t) apr_dir_read(a
>          {
>              apr_status_t rv;
>              char *fname;
>         +    apr_wchar_t wdirname[APR_PATH_MAX];
>
>
>     That's a crazy huge stack frame. I recognize we were doing this
>     before this commit, but is it advisable? Should it be corrected?
>
>     Cheers,
>     -g
>


Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by William A Rowe Jr <wr...@rowe-clan.net>.
I think the optimal way is to allocate a pair of apr thread-specific wchar
buffers in each thread's pool on startup, and use those exclusively
per-thread for wchar translations. We could be looking at 64k/thread
exclusively for name translation, but it doesn't seem unreasonable.

The alternative is to continue to use stack, we surely don't want to lock
on acquiring or allocating name translation buffers. /shrug





On Fri, Jun 7, 2019 at 6:27 AM Greg Stein <gs...@gmail.com> wrote:

> On Fri, Jun 7, 2019 at 6:02 AM <iv...@apache.org> wrote:
> >...
>
>> \@@ -114,116 +93,73 @@ APR_DECLARE(apr_status_t) apr_dir_read(a
>>  {
>>      apr_status_t rv;
>>      char *fname;
>> +    apr_wchar_t wdirname[APR_PATH_MAX];
>>
>
> That's a crazy huge stack frame. I recognize we were doing this before
> this commit, but is it advisable? Should it be corrected?
>
> Cheers,
> -g
>
>

Re: svn commit: r1860745 - /apr/apr/trunk/file_io/win32/dir.c

Posted by Greg Stein <gs...@gmail.com>.
On Fri, Jun 7, 2019 at 6:02 AM <iv...@apache.org> wrote:
>...

> \@@ -114,116 +93,73 @@ APR_DECLARE(apr_status_t) apr_dir_read(a
>  {
>      apr_status_t rv;
>      char *fname;
> +    apr_wchar_t wdirname[APR_PATH_MAX];
>

That's a crazy huge stack frame. I recognize we were doing this before this
commit, but is it advisable? Should it be corrected?

Cheers,
-g