X-Recipient:	archive-cygwin AT delorie DOT com
X-Spam-Check-By:	sourceware.org
From:	Jim Meyering <jim AT meyering DOT net>
To:	=?utf-8?Q?P=C3=A1draig?= Brady <P AT draigBrady DOT com>
Cc:	Eric Blake <ebb9 AT byu DOT net>, cygwin AT cygwin DOT com,
	bug-coreutils <bug-coreutils AT gnu DOT org>
Subject:	Re: "du -b --files0-from=-" running out of memory
In-Reply-To:	<492C3128.8000700@draigBrady.com> (=?utf-8?Q?=22P=C3=A1draig?= Brady"'s message of "Tue, 25 Nov 2008 17:08:56 +0000")
References:	<nacii4p76633jbufvfoj4qjesrph05rjga AT 4ax DOT com> <49296551 DOT 4010801 AT byu DOT net> <87bpw5a5tp DOT fsf AT rho DOT meyering DOT net> <874p1v52od DOT fsf AT rho DOT meyering DOT net> <492C1512 DOT 9020706 AT draigBrady DOT com> <87myfn3ft5 DOT fsf AT rho DOT meyering DOT net> <492C3128 DOT 8000700 AT draigBrady DOT com>
Date:	Tue, 25 Nov 2008 18:41:18 +0100
Message-ID:	<87bpw33dq9.fsf@rho.meyering.net>
Lines:	107
MIME-Version:	1.0
Mailing-List:	contact cygwin-help AT cygwin DOT com; run by ezmlm
List-Id:	<cygwin.cygwin.com>
List-Unsubscribe:	<mailto:cygwin-unsubscribe-archive-cygwin=delorie DOT com AT cygwin DOT com>
List-Subscribe:	<mailto:cygwin-subscribe AT cygwin DOT com>
List-Archive:	<http://sourceware.org/ml/cygwin/>
List-Post:	<mailto:cygwin AT cygwin DOT com>
List-Help:	<mailto:cygwin-help AT cygwin DOT com>, <http://sourceware.org/ml/#faqs>
Sender:	cygwin-owner AT cygwin DOT com
Mail-Followup-To:	cygwin AT cygwin DOT com
Delivered-To:	mailing list cygwin AT cygwin DOT com
X-MIME-Autoconverted:	from quoted-printable to 8bit by delorie.com id mAPHg9F6003548

Pádraig Brady <P AT draigBrady DOT com> wrote:
>>> I notice that argv_iter does a malloc() + memcpy() per entry.
>>> Since the sources are already NUL terminated strings
>>> perhaps it could just return a pointer to a getdelim
>>> realloc'd buffer which was referenced in the argv_iterator struct.
>>
>> The only per-entry allocation I see is:
>>   - in argv-mode: strdup
>>   - in stream-reading mode: getdelim
>>
>> Did I miss something?
>
> Sorry I was unclear.
> strdup() and getdelim(NULL,...) do a malloc() + memcpy()
> if you maintain the char* name and size_t buf_len in
> the argv_iterator struct, then you can return pointers
> to the orig data, and remove the need to free() from the
> users of argv_iter().

Good suggestion!  Thanks.

diff --git a/gl/lib/argv-iter.c b/gl/lib/argv-iter.c
index 204769a..adaff6f 100644
--- a/gl/lib/argv-iter.c
+++ b/gl/lib/argv-iter.c
@@ -28,6 +28,8 @@ struct argv_iterator
   /* file-mode: fp records position */
   FILE *fp;
   size_t item_idx;
+  char *tok;
+  size_t buf_len;

   /* argv-mode: record just argv and current pointer */
   char **arg_list;
@@ -55,6 +57,9 @@ argv_iter_init_stream (FILE *fp)
   if (!ai)
     return NULL;
   ai->fp = fp;
+  ai->tok = NULL;
+  ai->buf_len = 0;
+
   ai->item_idx = 0;
   ai->arg_list = NULL;
   return ai;
@@ -65,19 +70,16 @@ argv_iter (struct argv_iterator *ai, enum argv_iter_err *err)
 {
   if (ai->fp)
     {
-      char *name = NULL;
-      size_t buf_len = 0;
-      ssize_t len = getdelim (&name, &buf_len, '\0', ai->fp);
+      ssize_t len = getdelim (&ai->tok, &ai->buf_len, '\0', ai->fp);
       if (len < 0)
         {
-          free (name);
           *err = feof (ai->fp) ? AI_ERR_EOF : AI_ERR_READ;
           return NULL;
         }

       *err = AI_ERR_OK;
       ai->item_idx++;
-      return name;
+      return ai->tok;
     }
   else
     {
@@ -89,7 +91,7 @@ argv_iter (struct argv_iterator *ai, enum argv_iter_err *err)
       else
         {
           *err = AI_ERR_OK;
-          return strdup (*(ai->p++));
+          return *(ai->p++);
         }
     }
 }
@@ -103,6 +105,8 @@ argv_iter_n_args (struct argv_iterator const *ai)
 void
 argv_iter_free (struct argv_iterator *ai)
 {
+  if (ai->fp)
+    free (ai->tok);
   free (ai);
 }

diff --git a/src/du.c b/src/du.c
index 5ed2b12..a1345a7 100644
--- a/src/du.c
+++ b/src/du.c
@@ -968,7 +968,7 @@ main (int argc, char **argv)
 	    case AI_ERR_READ:
 	      error (0, errno, _("%s: read error"), quote (files_from));
 	      skip_file = true;
-	      goto next_file_name;
+	      continue;

 	    case AI_ERR_MEM:
 	      xalloc_die ();
@@ -1016,9 +1016,6 @@ main (int argc, char **argv)
 	  temp_argv[0] = file_name;
 	  ok &= du_files (temp_argv, bit_flags);
 	}
-
-    next_file_name:;
-      free (file_name);
     }

   argv_iter_free (ai);

--
Unsubscribe info:      http://cygwin.com/ml/#unsubscribe-simple
Problem reports:       http://cygwin.com/problems.html
Documentation:         http://cygwin.com/docs.html
FAQ:                   http://cygwin.com/faq/

- Raw text -