Mailing-List: contact cygwin-help AT cygwin DOT com; run by ezmlm
List-Subscribe: <mailto:cygwin-subscribe AT cygwin DOT com>
List-Archive: <http://sources.redhat.com/ml/cygwin/>
List-Post: <mailto:cygwin AT cygwin DOT com>
List-Help: <mailto:cygwin-help AT cygwin DOT com>, <http://sources.redhat.com/ml/#faqs>
Sender: cygwin-owner AT cygwin DOT com
Mail-Followup-To: cygwin AT cygwin DOT com
Delivered-To: mailing list cygwin AT cygwin DOT com
Message-ID: <3CCEA394.8000909@scytek.de>
Date: Tue, 30 Apr 2002 16:00:52 +0200
From: Volker Quetschke <quetschke AT scytek DOT de>
User-Agent: Mozilla/5.0 (Windows; U; Win98; de-DE; rv:0.9.4) Gecko/20011019 Netscape6/6.2
X-Accept-Language: de-DE
MIME-Version: 1.0
To: cygwin AT cygwin DOT com
Subject: Re: Using a real mirroring tool...
References: <FC169E059D1A0442A04C40F86D9BA7600C5F42 AT itdomain003 DOT itdomain DOT net DOT au> <NCBBIHCHBLCMLBLOBONKKEOLCNAA DOT g DOT r DOT vansickle AT worldnet DOT att DOT net> <uoiocu8tid9en854cjelr56do40nrp5dj7 AT 4ax DOT com> <3CCC7290 DOT 706 AT ece DOT gatech DOT edu> <3CCD971A DOT 5050205 AT ece DOT gatech DOT edu> <3CCD9F4D DOT 20209 AT ece DOT gatech DOT edu>
Content-Type: multipart/mixed;
 boundary="------------030604020207050904080907"
X-Sender: 320071185708-0001 AT t-dialin DOT net

--------------030604020207050904080907
Content-Type: text/plain; charset=us-ascii; format=flowed
Content-Transfer-Encoding: 7bit

Hi!

Charles Wilson wrote:

> A few notes:
> 
> ...
> 
> rsync can't merge.


Yes, but Michael A. Chases clean_setup.pl can! I allowed myself to add a 
new option to clean_setup.pl, it exports the list of missing files to a 
file. Then you can use wget to get all the missing files.

I attached the modified clean_setup.pl plus a shell scrip to get the 
missing files from a mirror. You have to modify the getmissing.sh to 
choose your mirror and your target download directory. The directory and 
both files must exist the same directory as your setup.exe. Just a quick 
hack, I guarantee nothing!

Bye
    Volker

--------------030604020207050904080907
Content-Type: text/plain;
 name="getmissing.sh"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="getmissing.sh"

#!/bin/bash

# Mirror script for cygwin
# Uses Michael A. Chase "cleanup_setup.pl"
# in a modified version (V1.0303)
# and wget

# getmissing.sh and cleanup_setup.pl must reside in the same directory
# as setup.exe

# Files to download
filelist="missing.lst"
# Choose your mirror with directory of setup.ini
mirror="ftp://ftp.inf.tu-dresden.de/software/windows/cygwin32/"
# Cut directories                   1        2       3
cutdir=3
# Name your download directory
mirrordir="_mymirror"

wget --mirror --no-host-directories --cut-dirs=$cutdir --passive-ftp -P $mirrordir ${mirror}setup.ini
./clean_setup.pl -Arch -writelist
wget --mirror --no-host-directories --cut-dirs=$cutdir --passive-ftp -P $mirrordir -i $filelist -B $mirror
./clean_setup.pl -Setup

--------------030604020207050904080907
Content-Type: text/plain;
 name="clean_setup.pl"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="clean_setup.pl"

#! /usr/bin/perl -w

require 5.005;

# Delete old files from Cygwin setup.exe archive subdirectories

# Name of file:  cleanup_setup.pl
#
# Type of file:  Perl script
#
# Author:        Michael A. Chase
#
# Purpose:       Clean out setup.exe archive subdirectories.

# Syntax:        See $HelpText Below
#--------------------------  MODIFICATION HISTORY -----------------------------
$VERSION = '1.0303';
# 020430 V. Quetschke   Add -writelist
#                       Writes list of missing files to missing.lst.
# 020423 M. Chase       Add -source, -install, -H options.
#                       Removed directory names from Missing Files list.
# $VERSION = '1.0301';
# 020417 M. Chase       Stop lc()ing file names during setup.ini parsing.
# $VERSION = '1.0300';
# 020416 M. Chase       Ignore setup.ini while collecting archive file names.
#                       Remove directories in reverse length order.
# $VERSION = '1.0202';
# 020416 M. Chase       Properly ignore directories in obsolete file search.
# $VERSION = '1.0201';
# 020404 M. Chase       Ignore files found more than once in move loop.
#                       Protect against attempts to move files from different
#                          trees over each other in the base tree.
# $VERSION = '1.0200';
# 020326 M. Chase       Properly handle directory case.
#                       Optionally move archives to base directory tree.
# $VERSION = '1.0100';
# 020212 M. Chase       Handle multiple setup.ini files and file sources.
# $VERSION = '1.0000';
# 010422 M. Chase       First draft.

use FindBin qw( $RealBin $RealScript );
use File::Basename qw( &basename &dirname &fileparse );
use File::Find;
use File::Spec;
use File::Path qw( &mkpath );
use File::Copy qw( &copy &move );
use Getopt::Long;

use strict;
use integer;
use vars qw( $VERSION );

# Initialize options
my $bArch    = 0;
my $bMove    = 0;
my $bSetup   = 0;
my $bCurr    = 1;
my $bPrev    = 1;
my $bTest    = 1;
my $bExp     = 1;
my $bInstall = 1;
my $bSource  = 0;
my $bWrite   = 0;
my @sHide    = ();
my @sIgnore  = ();
my $sDir0    = File::Spec -> rel2abs( "." );

# Syntax description
sub usage {
   my ( $sOpt, $sVal, @sMsg ) = @_;

   my $sHelpText = <<END_HELP_TEXT;
Cleanup Cygwin setup.exe package cache directories
syntax: $RealScript [opt]
Opt: ($VERSION)
   -[no]Move    = [Don't] Move archive files to base directory tree ($bMove)
   -[no]Arch    = [Don't] Delete obsolete archives and directories ($bArch)
   -[no]Setup   = [Don't] Delete obsolete setup.ini files ($bSetup), forces -Move
   -[no]curr    = [Don't] Report missing [curr] files ($bCurr)
   -[no]prev    = [Don't] Report missing [prev] files ($bPrev)
   -[no]test    = [Don't] Report missing [test] files ($bTest)
   -[no]exp     = [Don't] Report missing [exp]  files ($bExp)
   -[no]install = [Don't] Report missing install archives ($bInstall)
   -[no]source  = [Don't] Report missing source archives ($bSource)
   -writelist   = Write missing files report (missing.lst) to disk ($bWrite)
   -base dir    = Archive cache base directory ($sDir0)
   -I mask      = Ignore files and directories that match mask, multiple allowed
   -H mask      = Ignore packages that match mask, multiple allowed
      mask is a filename wildcard mask, not a regular expression
Arg:
END_HELP_TEXT
# Balance quotes in here document # ' # "

   my $nRet = 'help' eq $sOpt ? 0 : 0 + $sVal;
   select STDERR if $nRet;
   foreach ( @sMsg, $sHelpText ) { s/\s+$//; print "$_\n"; }
   exit $nRet;
}

# Parse command line
Getopt::Long::config( qw( no_ignore_case no_auto_abbrev require_order ) );
GetOptions(
   'Arch!'       => \$bArch,
   'Move!'       => \$bMove,
   'Setup!'      => \$bSetup,
   'curr!'       => \$bCurr,
   'prev!'       => \$bPrev,
   'test!'       => \$bTest,
   'exp!'        => \$bExp,
   'install!'    => \$bInstall,
   'source!'     => \$bSource,
   'writelist!'  => \$bWrite,
   'base=s'      => \$sDir0,
   'Hide|H=s@'   => \@sHide,
   'Ignore|I=s@' => \@sIgnore,
   'help|h' => \&usage ) or usage( 'die', 1 );
$bMove ||= $bSetup; # Don't remove setup files unless also moving archves
chdir $sDir0 or usage( 'die', 1, "Can't change directory to $sDir0, $!\n" );

# Provide default values for unset options and parameters

# Report arguments
$sDir0 = File::Spec -> rel2abs( "." );
$sDir0 =~ s,\\,/,g;
my $sOpt = '';
$sOpt .= "\nMoving archives to base directory tree" if $bMove;
$sOpt .= "\nDeleting unused files and empty directores" if $bArch;
$sOpt .= "\nDeleting obsolete setup.ini files" if $bSetup;
$sOpt .= "\nIgnoring files and directories: " . join ' ',  map { "'$_'" }
   @sIgnore if @sIgnore;
$sOpt .= "\nIgnoring packages: "   . join ' ',  map { "'$_'" } @sHide if @sHide;
print <<HERE;
Base Directory: $sDir0$sOpt
HERE

# Build file or directory name matcher
#    Adapted from Recipe 6.10 in Perl Cookbook
sub rfMatch {
   my ( $bDefault, $op, @sMask ) = @_; # @sMask must be a lexical array
   return sub { return $bDefault; } if 3 > @_;
   my $sExpr = join " $op\n",
      map {
         # Convert file expansion mask to regular expression
         $sMask[$_] =~ s/\./\\./g;
         $sMask[$_] =~ s/\?/.?/g;
         $sMask[$_] =~ s/\*/.*/g;
         "m:^\$sMask[$_]\$:oi";
         } 0 .. $#sMask;
   my $rfMatch = eval "sub { local \$_ = shift;\nreturn $sExpr; };";
   die $@ if $@;
   return $rfMatch;
}
local *bHide   = rfMatch( 0, '||', @sHide );
local *bIgnore = rfMatch( 0, '||', @sIgnore );

# Find and parse setup.ini files, collect other filenames at the same time
my ( %aSetup, $sRel, $sName, %sTarBall, %bDir );
my $wanted = sub {
   # Skip ., .., and files and directories in ignore list
   if ( '.' eq $_ || '..' eq $_ ) {
      $File::Find::prune = 1 if '..' eq $_;
      return;
   }
   if ( bIgnore( $_ ) ) {
      $File::Find::prune = 1 if -d $_; # Prune if a directory
      return;
   }

   # Handle directory or file
   $sRel = sRel( $File::Find::dir, $sDir0 );
   if ( -d $_ ) { $bDir{sRel( $File::Find::name, $sDir0 )} = 1; }
      # Remember directory name for possible removal
   elsif ( "setup.ini" eq $_ ) {
      # Parse setup.ini
      my @aSetup;

      # Get list of files to leave alone, includes subdirectory path
      # setup-timestamp: 1012849221
      # install: latest/bash/bash-2.05-1.tar.gz 576828
      # source:  latest/bash/bash-2.05-1-src.tar.gz 1792319
      my ( $bHide, $sGroup, $sType, $sFile, $sSize, $sVol, $sSubDir, $sName );
      my $sSetup = $File::Find::name;
      open( SETUP, $sSetup ) or usage( 'die', 1, "Can't open $sSetup, $!" );
      while ( <SETUP> ) {
         ( $sType, $sFile, $sSize )  = split /\s+/, $_;
         if    ( 'setup-timestamp:' eq $sType ) {
            $aSetup[0]         = $sFile; # Actually timestamp
         }
         elsif ( 'install:'         eq $sType ) {
            next if $bHide;
            ( $sVol, $sSubDir, $sName ) = File::Spec -> splitpath( $sFile );
            $sSubDir                    = File::Spec -> canonpath( $sSubDir );
            $aSetup[1]{$sName}          = [ $sGroup, $sSize, $sSubDir ];
         }
         elsif ( 'source:'          eq $sType ) {
            next if $bHide;
            ( $sVol, $sSubDir, $sName ) = File::Spec -> splitpath( $sFile );
            $sSubDir                    = File::Spec -> canonpath( $sSubDir );
            $aSetup[2]{$sName}          = [ $sGroup, $sSize, $sSubDir ];
         }
         elsif ( '[' eq substr( $_, 0, 1 ) )    { s/\s+$//; $sGroup = $_; } # ]
         elsif ( s/^@\s+// ) {
            s/\s+$//;
            $bHide  = bHide( $_ );
            $sGroup = '[curr]';
         }
      }
      close SETUP;
      usage( 'die', 1, "Nothing found in $sSetup" )
         if 3 != @aSetup || ! $aSetup[0];
      $aSetup{$sRel} = \@aSetup;
   }
   elsif ( ".tar.bz2" eq substr( $_, -8 ) || ".tar.gz" eq substr( $_, -7 ) ) {
      # Save name of archive file
      $sTarBall{$_}{File::Spec -> canonpath( $sRel )} = -s $_;
   }
   # Currently ignoring other types of files
};
find( $wanted, $sDir0 );
usage( "die", 1, "No setup.ini files found" ) if ! %aSetup;

# Pick newest setup.ini
my ( $tNewest, $sNewest, @sOldSetup ) = ( 0, "" );
foreach ( sort keys %aSetup ) {
   if ( $tNewest < $aSetup{$_}[0] ) {
      $tNewest = $aSetup{$_}[0];
      $sNewest = $_;
   }
}
print "Newest setup.ini files: ", scalar gmtime( $tNewest ), " GMT\n";
foreach ( sort keys %aSetup ) {
   print "   ", sUnPercent( sRel( $_ ) ), "\n" if $tNewest == $aSetup{$_}[0];
}

# Report or remove obsolete setup.ini files
foreach ( sort keys %aSetup ) {
   push @sOldSetup, $_ if $tNewest > $aSetup{$_}[0];
}
print "Obsolete setup.ini files:\n" if @sOldSetup;
foreach ( @sOldSetup ) {
   print "   ", scalar gmtime( $aSetup{$_}[0] ), " GMT ",
     sUnPercent( sRel( $_ ) ), "\n";
   unlink "$_/setup.ini" or print "      *** Can't remove, $!" if $bSetup;
}

# Check found files against those listed in latest setup.ini
my ( @sDir, $sDir, $sFile, @sDup, %sMove, @sRemove, @sWrongSize );
my %aInstall = ( %{$aSetup{$sNewest}[1]} );
my %aSource  = ( %{$aSetup{$sNewest}[2]} );
foreach $sName ( sort keys %sTarBall ) {
   @sDir  = sort keys %{$sTarBall{$sName}};
   $sDir  = $sDir[0];
   $sFile = File::Spec -> catfile( $sDir, $sName );
   if    ( 1 < @sDir ) {
      # More than one copy of a file was found
      push @sDup, "$sName: " . join( ", ", map { sUnPercent( $_ ) } @sDir );
   }
   elsif ( exists $aInstall{$sName} ) {
      if    ( $sDir eq $aInstall{$sName}[2] ) {} # Already in right place
      elsif ( $sTarBall{$sName}{$sDir} != $aInstall{$sName}[1] ) {
         # Wrong size
         push @sWrongSize,
            "$sFile: $sTarBall{$sName}{$sDir} != $aInstall{$sName}[1] in " .
            sUnPercent( $sDir );
      }
      else {
         $sMove{$sFile} = File::Spec -> catfile( $aInstall{$sName}[2], $sName );
      }
   }
   elsif ( exists $aSource{$sName} ) {
      if    ( $sDir eq $aSource{$sName}[2] ) {} # Already in right place
      elsif ( $sTarBall{$sName}{$sDir} != $aSource{$sName}[1] ) {
         # Wrong size
         push @sWrongSize,
            "$sFile: $sTarBall{$sName}{$sDir} != $aSource{$sName}[1] in " .
            sUnPercent( $sDir );
      }
      else {
         $sMove{$sFile} = File::Spec -> catfile( $aSource{$sName}[2], $sName );
      }
   }
   else {
      # File not in setup.ini
      push @sRemove, $sFile;
   }
}

# Check for missing files
my @sMissing = ();
my @sMissingList = ();

if ( $bInstall ) {
  foreach ( keys %aInstall ) {
	if ( ! exists $sTarBall{$_} &&
	  ( $bCurr || '[curr]' ne $aInstall{$_}[0] ) &&
	  ( $bPrev || '[prev]' ne $aInstall{$_}[0] ) &&
      ( $bTest || '[test]' ne $aInstall{$_}[0] ) &&
	  ( $bExp  || '[exp]'  ne $aInstall{$_}[0] ) ) {
		push @sMissing, "$aInstall{$_}[0] $aInstall{$_}[2]/$_" ;
		  # Alternative: $aInstall{$_}[0] . " " .
		  #              File::Spec -> catfile( $aInstall{$_}[2],  $_ ) }
		push @sMissingList, "$aInstall{$_}[2]/$_" ;
	  }
  }
}
if ( $bSource ) {
  foreach ( keys %aSource ) {
	if ( ! exists $sTarBall{$_} &&
	  ( $bCurr || '[curr]' ne $aSource{$_}[0] ) &&
      ( $bPrev || '[prev]' ne $aSource{$_}[0] ) &&
      ( $bTest || '[test]' ne $aSource{$_}[0] ) &&
      ( $bExp  || '[exp]'  ne $aSource{$_}[0] ) ) {
		push @sMissing, "$aSource{$_}[0] $_" ;
		  # Alternative: $aSource{$_}[0] . " " .
		  #              File::Spec -> catfile( $aSource{$_}[2],  $_ ) }
		push @sMissingList, "$aSource{$_}[2]/$_" ;
	  }
  }
}
@sMissing = sort @sMissing;

# Complain about errors
print join( "\n   ", "\nDuplicate Files",  @sDup ),       "\n" if @sDup;
print join( "\n   ", "\nWrong Size Files", @sWrongSize ), "\n" if @sWrongSize;
print join( "\n   ", "\nMissing Files:",   @sMissing ),   "\n" if @sMissing;

# Report list of missing files to disk
if ( $bWrite ) {
  @sMissingList = sort @sMissingList;
  open(REPORT, '>', 'missing.lst');
  print( REPORT join( "\n", @sMissingList ),"\n" ) if @sMissingList;
  close(REPORT);
}

# Move queued files to base directory tree
if ( %sMove ) {
   print $bMove ? "\n" : "\nNot ", "Moving files to base directory tree\n";
   my ( $sFrom, $sTo );
   foreach $sFrom ( sort keys %sMove ) {
      $sTo  = $sMove{$sFrom};
      $sDir = dirname( $sTo );
      print sUnPercent( "   $sFrom -> $sDir\n" );
      if ( -e $sTo ) { print "      *** Target already exists\n"; }
      elsif ( $bMove ) {
         mkpath( $sDir ) if ! -d $sDir;
         move( $sFrom, $sTo ) or print "      *** Can't move, $!\n";
      }
   }
}

# Remove files not listed in setup.ini
if ( @sRemove ) {
   print $bArch ? "\n" : "\nNot ", "Removing files not in setup.ini\n";
   foreach $sFile ( @sRemove ) {
      print sUnPercent( "   $sFile\n" );
      if ( $bArch ) {
         unlink( $sFile ) or print "      *** Can't remove, $!\n";
      }
   }
}

# Remove empty directories
my ( @g, $bFound );
foreach $sRel ( sort { length $b <=> length $a || $a cmp $b } keys %bDir ) {
   @g = glob( File::Spec -> catfile( $sRel, "*" ) );
   if ( ! @g ) {
      print $bArch ? "\n" : "\nNot ", "Removing Empty Directories\n"
         if ! $bFound++;
      print sUnPercent( "   $sRel\n" );
      if ( $bArch ) {
         rmdir $sRel or print "      *** Can't rmdir, $!\n";
      }
   }
}

# Copy latest setup.ini to base directory
if ( "." ne $sNewest ) {
   print sUnPercent( "\nCopying setup.ini from $sNewest to $sDir0\n" );
   copy( "$sNewest/setup.ini", "$sDir0/setup.ini" )
      or print "   *** Can't copy setup.ini, $!\n";
}

exit 0;

# Produce relative file or directory
sub sRel {
   my ( $sAbs, $sBase ) = @_;
   $sBase = "." if ! defined $sBase || ! length $sBase;
   my $sRel = File::Spec -> abs2rel( $sAbs, $sBase );
   $sRel =~ s,\\,/,g;
   $sRel =~ s/^\w://;
   $sRel =  "." if ! length $sRel;
   return $sRel;
}

# Convert %xx to characters
sub sUnPercent {
   local ( $_ ) = @_;

   s/\%([0-9a-f]{2})/chr(hex($1))/gie;
   return $_;
}


--------------030604020207050904080907
Content-Type: text/plain; charset=us-ascii

--
Unsubscribe info:      http://cygwin.com/ml/#unsubscribe-simple
Bug reporting:         http://cygwin.com/bugs.html
Documentation:         http://cygwin.com/docs.html
FAQ:                   http://cygwin.com/faq/
--------------030604020207050904080907--