[PATCH] Fix calloc() behaviour in case of overflow

[Gedare Bloom]

Looks good.

On Tue, Apr 20, 2021, 12:33 PM Sebastian Huber <
sebastian.huber at embedded-brains.de> wrote:

> The multiplication to calculate the length of the memory area to
> allocate may overflow.  Return NULL in case of an overflow.
>
> Close #4389.
> ---
>  cpukit/libcsupport/src/calloc.c       | 13 ++++++++++++-
>  cpukit/libcsupport/src/rtemscalloc.c  |  9 ++++++++-
>  testsuites/libtests/malloctest/init.c | 23 +++++++++++++----------
>  3 files changed, 33 insertions(+), 12 deletions(-)
>
> diff --git a/cpukit/libcsupport/src/calloc.c
> b/cpukit/libcsupport/src/calloc.c
> index e015f30d6c..693aa21453 100644
> --- a/cpukit/libcsupport/src/calloc.c
> +++ b/cpukit/libcsupport/src/calloc.c
> @@ -20,7 +20,10 @@
>
>  #if defined(RTEMS_NEWLIB) && !defined(HAVE_CALLOC)
>  #include <stdlib.h>
> +
> +#include <errno.h>
>  #include <string.h>
> +
>  #include <rtems/score/basedefs.h>
>
>  void *calloc(
> @@ -31,7 +34,15 @@ void *calloc(
>    void   *cptr;
>    size_t  length;
>
> -  length = nelem * elsize;
> +  if ( nelem == 0 ) {
> +    length = 0;
> +  } else if ( elsize > SIZE_MAX / nelem ) {
> +    errno = ENOMEM;
> +    return NULL;
> +  } else {
> +    length = nelem * elsize;
> +  }
> +
>    cptr = malloc( length );
>    RTEMS_OBFUSCATE_VARIABLE( cptr );
>    if ( RTEMS_PREDICT_FALSE( cptr == NULL ) ) {
> diff --git a/cpukit/libcsupport/src/rtemscalloc.c
> b/cpukit/libcsupport/src/rtemscalloc.c
> index 4e189e8367..836f1da64d 100644
> --- a/cpukit/libcsupport/src/rtemscalloc.c
> +++ b/cpukit/libcsupport/src/rtemscalloc.c
> @@ -46,7 +46,14 @@ void *rtems_calloc( size_t nelem, size_t elsize )
>    size_t  length;
>    void   *p;
>
> -  length = nelem * elsize;
> +  if ( nelem == 0 ) {
> +    length = 0;
> +  } else if ( elsize > SIZE_MAX / nelem ) {
> +    return NULL;
> +  } else {
> +    length = nelem * elsize;
> +  }
> +
>    p = rtems_malloc( length );
>    RTEMS_OBFUSCATE_VARIABLE( p );
>    if ( RTEMS_PREDICT_FALSE( p == NULL ) ) {
> diff --git a/testsuites/libtests/malloctest/init.c
> b/testsuites/libtests/malloctest/init.c
> index 1d91385683..4d0f421c02 100644
> --- a/testsuites/libtests/malloctest/init.c
> +++ b/testsuites/libtests/malloctest/init.c
> @@ -1190,6 +1190,14 @@ static void test_rtems_calloc(void)
>    rtems_test_assert(p == NULL);
>    rtems_test_assert(errno == 0);
>
> +#pragma GCC diagnostic push
> +#pragma GCC diagnostic ignored "-Walloc-size-larger-than=N"
> +  errno = 0;
> +  p = rtems_calloc(SIZE_MAX, SIZE_MAX);
> +  rtems_test_assert(p == NULL);
> +  rtems_test_assert(errno == 0);
> +#pragma GCC diagnostic pop
> +
>    i = rtems_calloc(1, sizeof(*i));
>    rtems_test_assert(i != NULL);
>    rtems_test_assert(*i == 0);
> @@ -1313,22 +1321,17 @@ rtems_task Init(
>  #pragma GCC diagnostic push
>  #pragma GCC diagnostic ignored "-Walloc-size-larger-than=N"
>    p1 = calloc( 1, SIZE_MAX );
> +  rtems_test_assert( p1 == NULL );
> +
> +  p1 = calloc( SIZE_MAX, SIZE_MAX );
> +  rtems_test_assert( p1 == NULL );
>  #pragma GCC diagnostic pop
> -  if (p1) {
> -    printf("ERROR on attempt to calloc SIZE_MAX block expected failure.");
> -    free( p1 );
> -  }
>
>    /*
>     * Verify error case where malloc of size 0.
>     */
>    p1 = malloc( 0 );
> -  if (p1) {
> -    printf("ERROR on attempt to malloc size 0 block expected failure.");
> -    free( p1 );
> -  }
> -
> -
> +  rtems_test_assert( p1 == NULL );
>
>    test_heap_initialize();
>    test_heap_block_allocate();
> --
> 2.26.2
>
> _______________________________________________
> devel mailing list
> devel at rtems.org
> http://lists.rtems.org/mailman/listinfo/devel
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.rtems.org/pipermail/devel/attachments/20210420/d55edffa/attachment.html>