main_rtrace.c: Add error return when malloc fails

CID 1399709: Dereference after null check in
rtems_trace_buffering_shell_save().

Closes #4329

cpukit/libmisc/shell/main_rtrace.c

@@ -473,6 +473,7 @@ rtems_trace_buffering_shell_save (int argc, char *argv[])
   {
     close (out);
     printf ("error: no memory\n");
+    return 1;
   }
 
   memset (buf, 0, SAVE_BUF_SIZE);