3 # debianqueued -- daemon for managing Debian upload queues
5 # Copyright (C) 1997 Roman Hodek <Roman.Hodek@informatik.uni-erlangen.de>
6 # Copyright (C) 2001-2007 Ryan Murray <rmurray@debian.org>
7 # Copyright (C) 2008 Thomas Viehmann <tv@beamnet.de>
9 # This program is free software. You can redistribute it and/or
10 # modify it under the terms of the GNU General Public License as
11 # published by the Free Software Foundation: either version 2 or
12 # (at your option) any later version.
13 # This program comes with ABSOLUTELY NO WARRANTY!
19 use POSIX qw( strftime sys_stat_h sys_wait_h signal_h );
22 use Socket qw( PF_INET AF_INET SOCK_STREAM );
28 setlocale(&POSIX::LC_ALL, "C");
30 # ---------------------------------------------------------------------------
32 # ---------------------------------------------------------------------------
35 ( $conf::queued_dir = ( ( $0 !~ m,^/, ) ? POSIX::getcwd() . "/" : "" ) . $0 )
37 require "$conf::queued_dir/config";
38 my $junk = $conf::debug; # avoid spurious warnings about unused vars
39 $junk = $conf::ssh_key_file;
40 $junk = $conf::stray_remove_timeout;
41 $junk = $conf::problem_report_timeout;
42 $junk = $conf::queue_delay;
43 $junk = $conf::keep_files;
44 $junk = $conf::valid_files;
45 $junk = $conf::max_upload_retries;
46 $junk = $conf::upload_delay_1;
47 $junk = $conf::upload_delay_2;
51 $junk = $conf::check_md5sum;
55 $junk = $conf::ftpdebug;
56 $junk = $conf::ftptimeout;
57 $junk = $conf::no_changes_timeout;
58 $junk = @conf::nonus_packages;
59 $junk = @conf::test_binaries;
60 $junk = @conf::maintainer_mail;
61 $junk = @conf::targetdir_delayed;
62 $junk = $conf::mail ||= '/usr/sbin/sendmail';
63 $junk = $conf::overridemail;
64 $conf::target = "localhost" if $conf::upload_method eq "copy";
68 ( $main::progname = $0 ) =~ s,.*/,,;
70 ($main::hostname, undef, undef, undef, undef) = gethostbyname(hostname());
73 my $re_file_safe_prefix = qr/\A([a-zA-Z0-9][a-zA-Z0-9_.:~+-]*)/s;
74 my $re_file_safe = qr/$re_file_safe_prefix\z/s;
76 # extract -r and -k args
78 if ( @ARGV == 1 && $ARGV[0] =~ /^-[rk]$/ ) {
79 $main::arg = ( $ARGV[0] eq '-k' ) ? "kill" : "restart";
83 # test for another instance of the queued already running
84 my ( $pid, $delayed_dirs, $adelayedcore );
85 if ( open( PIDFILE, "<", $conf::pidfile ) ) {
86 chomp( $pid = <PIDFILE> );
90 # remove stale pid file
91 unlink($conf::pidfile);
92 } elsif ($main::arg) {
94 print "Killing running daemon (pid $pid) ...";
97 while ( kill( 0, $pid ) && $cnt-- > 0 ) {
101 if ( kill( 0, $pid ) ) {
102 print " failed!\nProcess $pid still running.\n";
106 if ( -e "$conf::incoming/core" ) {
107 unlink("$conf::incoming/core");
108 print "(Removed core file)\n";
110 for ( $delayed_dirs = 0 ;
111 $delayed_dirs <= $conf::max_delayed ;
115 sprintf( "$conf::incoming_delayed/core", $delayed_dirs );
116 if ( -e $adelayedcore ) {
117 unlink($adelayedcore);
118 print "(Removed core file)\n";
120 } ## end for ( $delayed_dirs = 0...
121 exit 0 if $main::arg eq "kill";
123 die "Another $main::progname is already running (pid $pid)\n"
124 if $pid && kill( 0, $pid );
126 } elsif ( $main::arg eq "kill" ) {
127 die "No daemon running\n";
128 } elsif ( $main::arg eq "restart" ) {
129 print "(No daemon running; starting anyway)\n";
132 # if started without arguments (initial invocation), then fork
135 # now go to background
136 die "$main::progname: fork failed: $!\n"
137 unless defined( $pid = fork );
140 # parent: wait for signal from child (SIGCHLD or SIGUSR1) and exit
141 my $sigset = POSIX::SigSet->new();
143 $SIG{"CHLD"} = sub { };
144 $SIG{"USR1"} = sub { };
145 POSIX::sigsuspend($sigset);
146 waitpid( $pid, WNOHANG );
147 if ( kill( 0, $pid ) ) {
148 print "Daemon (on $main::hostname) started in background (pid $pid)\n";
157 if ( $conf::upload_method eq "ssh" ) {
159 # exec an ssh-agent that starts us again
160 # force shell to be /bin/sh, ssh-agent may base its decision
161 # whether to use a fd or a Unix socket on the shell...
162 $ENV{"SHELL"} = "/bin/sh";
163 exec $conf::ssh_agent, $0, "startup", getppid();
164 die "$main::progname: Could not exec $conf::ssh_agent: $!\n";
167 # no need to exec, just set up @ARGV as expected below
168 @ARGV = ( "startup", getppid() );
170 } ## end else [ if ($pid)
171 } ## end if ( !@ARGV )
172 die "Please start without any arguments.\n"
173 if @ARGV != 2 || $ARGV[0] ne "startup";
174 my $parent_pid = $ARGV[1];
178 ( $version = 'Release: 0.95' ) =~ s/\$ ?//g;
179 print "debianqueued $version\n";
182 # check if all programs exist
184 foreach $prg ( $conf::gpg, $conf::ssh, $conf::scp, $conf::ssh_agent,
185 $conf::ssh_add, $conf::md5sum, $conf::mail, $conf::mkfifo )
187 die "Required program $prg doesn't exist or isn't executable\n"
190 # check for correct upload method
191 die "Bad upload method '$conf::upload_method'.\n"
192 if $conf::upload_method ne "ssh"
193 && $conf::upload_method ne "ftp"
194 && $conf::upload_method ne "copy";
195 die "No keyrings\n" if !@conf::keyrings;
197 } ## end foreach $prg ( $conf::gpg, ...
198 die "statusfile path must be absolute."
199 if $conf::statusfile !~ m,^/,;
200 die "upload and target queue paths must be absolute."
201 if $conf::incoming !~ m,^/,
202 || $conf::incoming_delayed !~ m,^/,
203 || $conf::targetdir !~ m,^/,
204 || $conf::targetdir_delayed !~ m,^/,;
206 # ---------------------------------------------------------------------------
208 # ---------------------------------------------------------------------------
213 sub get_filelist_from_known_good_changes($);
214 sub age_delayed_queues();
215 sub process_changes($\@);
216 sub process_commands($);
217 sub age_delayed_queues();
218 sub is_on_target($\@);
219 sub copy_to_target(@);
222 sub check_incoming_writable();
224 sub write_status_file();
225 sub print_status($$$$$$);
226 sub format_status_num(\$$);
227 sub format_status_str(\$$);
239 sub check_incoming_writable();
242 sub is_debian_file($);
243 sub debian_file_stem($);
249 sub try_to_get_mail_addr($$);
253 sub unblock_signals();
256 sub restart_statusd();
259 $ENV{"PATH"} = "/bin:/usr/bin";
260 $ENV{"IFS"} = "" if defined( $ENV{"IFS"} && $ENV{"IFS"} ne "" );
273 sub ST_CTIME() { 10 }
275 # fixed lengths of data items passed over status pipe
276 sub STATNUM_LEN() { 30 }
277 sub STATSTR_LEN() { 128 }
279 # init list of signals
280 defined $Config{sig_name}
281 or die "$main::progname: No signal list defined!\n";
284 foreach $name ( split( ' ', $Config{sig_name} ) ) {
285 $main::signo{$name} = $i++;
288 @main::fatal_signals = qw( INT QUIT ILL TRAP ABRT BUS FPE USR2 SEGV PIPE
289 TERM XCPU XFSZ PWR );
291 $main::block_sigset = POSIX::SigSet->new;
292 $main::block_sigset->addset( $main::signo{"INT"} );
293 $main::block_sigset->addset( $main::signo{"TERM"} );
295 # some constant net stuff
296 $main::tcp_proto = ( getprotobyname('tcp') )[2]
297 or die "Cannot get protocol number for 'tcp'\n";
298 my $used_service = ( $conf::upload_method eq "ssh" ) ? "ssh" : "ftp";
299 $main::echo_port = ( getservbyname( $used_service, 'tcp' ) )[2]
300 or die "Cannot get port number for service '$used_service'\n";
302 # clear queue of stored mails
303 @main::stored_mails = ();
305 # run ssh-add to bring the key into the agent (will use stdin/stdout)
306 if ( $conf::upload_method eq "ssh" ) {
307 system "$conf::ssh_add $conf::ssh_key_file"
308 and die "$main::progname: Running $conf::ssh_add failed "
309 . "(exit status ", $? >> 8, ")\n";
312 # change to queue dir
313 chdir($conf::incoming)
314 or die "$main::progname: cannot cd to $conf::incoming: $!\n";
316 # needed before /dev/null redirects, some system send a SIGHUP when loosing
317 # the controlling tty
318 $SIG{"HUP"} = "IGNORE";
320 # open logfile, make it unbuffered
321 open( LOG, ">>", $conf::logfile )
322 or die "Cannot open my logfile $conf::logfile: $!\n";
323 chmod( 0644, $conf::logfile )
324 or die "Cannot set modes of $conf::logfile: $!\n";
325 select( ( select(LOG), $| = 1 )[0] );
328 $SIG{"HUP"} = \&close_log;
330 # redirect stdin, ... to /dev/null
331 open( STDIN, "<", "/dev/null" )
332 or die "$main::progname: Can't redirect stdin to /dev/null: $!\n";
333 open( STDOUT, ">&", \*LOG )
334 or die "$main::progname: Can't redirect stdout to $conf::logfile: $!\n";
335 open( STDERR, ">&", \*LOG )
336 or die "$main::progname: Can't redirect stderr to $conf::logfile: $!\n";
338 # ok, from this point usually no "die" anymore, stderr is gone!
339 msg( "log", "daemon (pid $$) (on $main::hostname) started\n" );
341 # initialize variables used by send_status before launching the status daemon
343 format_status_num( $main::next_run, time + 10 );
344 format_status_str( $main::current_changes, "" );
346 $main::incoming_writable = 1; # assume this for now
348 # start the daemon watching the 'status' FIFO
349 if ( $conf::statusfile && $conf::statusdelay == 0 ) {
350 $main::statusd_pid = fork_statusd();
351 $SIG{"CHLD"} = \&kid_died; # watch out for dead status daemon
352 # SIGUSR1 triggers status info
353 $SIG{"USR1"} = \&send_status;
354 } ## end if ( $conf::statusfile...
355 $main::maind_pid = $$;
358 kill( $main::signo{"ABRT"}, $$ )
359 if defined $main::signo{"ABRT"};
363 open( PIDFILE, ">", $conf::pidfile )
364 or msg( "log", "Can't open $conf::pidfile: $!\n" );
365 printf PIDFILE "%5d\n", $$;
367 chmod( 0644, $conf::pidfile )
368 or die "Cannot set modes of $conf::pidfile: $!\n";
370 # other signals will just log an error and exit
371 foreach (@main::fatal_signals) {
372 $SIG{$_} = \&fatal_signal;
375 # send signal to user-started process that we're ready and it can exit
376 kill( $main::signo{"USR1"}, $parent_pid );
378 # ---------------------------------------------------------------------------
380 # ---------------------------------------------------------------------------
382 # default to classical incoming/target
383 $main::current_incoming = $conf::incoming;
384 $main::current_targetdir = $conf::targetdir;
387 write_status_file() if $conf::statusdelay;
390 # ping target only if there is the possibility that we'll contact it (but
391 # also don't wait too long).
392 my @have_changes = <*.changes *.commands *.dak-commands>;
393 for ( my $delayed_dirs = 0 ;
394 $delayed_dirs <= $conf::max_delayed ;
397 my $adelayeddir = sprintf( "$conf::incoming_delayed", $delayed_dirs );
398 push( @have_changes, <$adelayeddir/*.changes> );
399 } ## end for ( my $delayed_dirs ...
401 if @have_changes || ( time - $main::last_ping_time ) > 8 * 60 * 60;
403 if ( @have_changes && $main::target_up ) {
404 check_incoming_writable if !$main::incoming_writable;
405 check_dir() if $main::incoming_writable;
408 write_status_file() if $conf::statusdelay;
410 if ( $conf::upload_method eq "copy" ) {
411 age_delayed_queues();
414 # sleep() returns if we received a signal (SIGUSR1 for status FIFO), so
415 # calculate the end time once and wait for it being reached.
416 format_status_num( $main::next_run, time + $conf::queue_delay );
418 while ( ( $delta = calc_delta() ) > 0 ) {
419 debug("mainloop sleeping $delta secs");
422 # check if statusd died, if using status FIFO, or update status file
423 if ($conf::statusdelay) {
428 } ## end while ( ( $delta = calc_delta...
434 $delta = $main::next_run - time;
435 $delta = $conf::statusdelay
436 if $conf::statusdelay && $conf::statusdelay < $delta;
438 } ## end sub calc_delta()
440 # ---------------------------------------------------------------------------
441 # main working functions
442 # ---------------------------------------------------------------------------
445 # main function for checking the incoming dir
448 my ( @files, @changes, @keep_files, @this_keep_files, @stats, $file,
451 debug("starting checkdir");
453 write_status_file() if $conf::statusdelay;
455 # test if needed binaries are available; this is if they're on maybe
456 # slow-mounted NFS filesystems
457 foreach (@conf::test_binaries) {
460 # maybe the mount succeeds now
463 msg( "log", "binary test failed for $_; delaying queue run\n" );
465 } ## end foreach (@conf::test_binaries)
467 for ( $adelay = -1 ; $adelay <= $conf::max_delayed ; $adelay++ ) {
468 if ( $adelay == -1 ) {
469 $main::current_incoming = $conf::incoming;
470 $main::current_incoming_short = "";
471 $main::current_targetdir = $conf::targetdir;
473 $main::current_incoming = sprintf( $conf::incoming_delayed, $adelay );
474 $main::current_incoming_short = sprintf( "DELAYED/%d-day", $adelay );
475 $main::current_targetdir = sprintf( $conf::targetdir_delayed, $adelay );
478 # need to clear directory specific variables
480 undef(@this_keep_files);
482 chdir($main::current_incoming)
486 "Cannot change to dir "
487 . "${main::current_incoming_short}: $!\n"
492 # look for *.commands and *.dak-commands files but not in delayed queues
493 if ( $adelay == -1 ) {
494 foreach $file (<*.commands>) {
497 process_commands($file);
500 write_status_file() if $conf::statusdelay;
502 } ## end foreach $file (<*.commands>)
503 foreach $file (<*.dak-commands>) {
506 process_dak_commands($file);
509 write_status_file() if $conf::statusdelay;
512 } ## end if ( $adelay == -1 )
516 "log", "Cannot open dir ${main::current_incoming_short}: $!\n"
520 @files = readdir(INC);
523 # process all .changes files found
524 @changes = grep /\.changes$/, @files;
525 push( @keep_files, @changes ); # .changes files aren't stray
526 foreach $file (@changes) {
529 # wrap in an eval to allow jumpbacks to here with die in case
532 eval { process_changes( $file, @this_keep_files ); };
534 msg( "log,mail", $@ ) if $@;
536 write_status_file() if $conf::statusdelay;
538 # files which are ok in conjunction with this .changes
539 debug("$file tells to keep @this_keep_files");
540 push( @keep_files, @this_keep_files );
543 # break out of this loop if the incoming dir has become unwritable
544 goto end_run if !$main::incoming_writable;
545 } ## end foreach $file (@changes)
546 ftp_close() if $conf::upload_method eq "ftp";
548 # find files which aren't related to any .changes
549 foreach $file (@files) {
551 # filter out files we never want to delete
552 next if !-f $file || # may have disappeared in the meantime
555 || ( grep { $_ eq $file } @keep_files )
556 || $file =~ /$conf::keep_files/;
558 # Delete such files if they're older than
559 # $stray_remove_timeout; they could be part of an
560 # yet-incomplete upload, with the .changes still missing.
561 # Cannot send any notification, since owner unknown.
562 next if !( @stats = stat($file) );
563 my $age = time - $stats[ST_MTIME];
564 my ( $maint, $pattern, @job_files );
565 if ( $file =~ /^junk-for-writable-test/
566 || $file !~ m,$conf::valid_files,
567 || $age >= $conf::stray_remove_timeout )
570 "Deleted stray file ${main::current_incoming_short}/$file\n" )
574 "found stray file ${main::current_incoming_short}/$file, deleting in ",
575 print_time( $conf::stray_remove_timeout - $age )
577 } ## end else [ if ( $file =~ /^junk-for-writable-test/...
578 } ## end foreach $file (@files)
579 } ## end for ( $adelay = -1 ; $adelay...
580 chdir($conf::incoming);
584 write_status_file() if $conf::statusdelay;
585 } ## end sub check_dir()
587 sub get_filelist_from_known_good_changes($) {
593 # parse the .changes file
594 open( CHANGES, "<", $changes )
595 or die "$changes: $!\n";
596 outer_loop: while (<CHANGES>) {
599 redo outer_loop if !/^\s/;
600 my @field = split(/\s+/);
603 # forbid shell meta chars in the name, we pass it to a
604 # subshell several times...
605 $field[5] =~ /$re_file_safe/;
606 if ( $1 ne $field[5] ) {
607 msg( "log", "found suspicious filename $field[5]\n" );
610 push( @filenames, $field[5] );
611 } ## end while (<CHANGES>)
612 } ## end if (/^Files:/i)
613 } ## end while (<CHANGES>)
616 } ## end sub get_filelist_from_known_good_changes($)
619 # process one .changes file
621 sub process_changes($\@) {
623 my $keep_list = shift;
625 $pgplines, @files, @filenames, @changes_stats,
626 $failure_file, $retries, $last_retry, $upload_time,
627 $file, $do_report, $ls_l, $problems_reported,
628 $errs, $pkgname, $signator, $extralines
633 format_status_str( $main::current_changes,
634 "$main::current_incoming_short/$changes" );
636 write_status_file() if $conf::statusdelay;
639 msg( "log", "processing ${main::current_incoming_short}/$changes\n" );
641 # parse the .changes file
642 open( CHANGES, "<", $changes )
643 or die "Cannot open ${main::current_incoming_short}/$changes: $!\n";
646 $main::mail_addr = "";
648 outer_loop: while (<CHANGES>) {
649 if (/^---+(BEGIN|END) PGP .*---+$/) {
653 if ( $pgplines < 1 or $pgplines >= 3 ) {
654 $extralines++ if length $_ > 1;
657 if (/^Maintainer:\s*/i) {
658 chomp( $main::mail_addr = $' );
659 $main::mail_addr = $1 if $main::mail_addr =~ /<([^>]*)>/;
660 } elsif (/^Source:\s*/i) {
661 chomp( $pkgname = $' );
662 $pkgname =~ s/\s+$//;
663 $main::packages{$pkgname}++;
664 } elsif (/^Files:/i) {
666 redo outer_loop if !/^\s/;
667 my @field = split(/\s+/);
670 # forbid shell meta chars in the name, we pass it to a
671 # subshell several times...
672 $field[5] =~ /$re_file_safe/;
673 if ( $1 ne $field[5] ) {
674 msg( "log", "found suspicious filename $field[5]\n" );
677 "File '$field[5]' mentioned in $main::current_incoming_short/$changes\n",
678 "has bad characters in its name. Removed.\n"
682 } ## end if ( $1 ne $field[5] )
691 push( @filenames, $field[5] );
692 debug( "includes file $field[5], size $field[2], ", "md5 $field[1]" );
693 } ## end while (<CHANGES>)
694 } ## end elsif (/^Files:/i)
695 } ## end while (<CHANGES>)
698 # tell check_dir that the files mentioned in this .changes aren't stray,
699 # we know about them somehow
700 @$keep_list = @filenames;
702 # some consistency checks
705 "$main::current_incoming_short/$changes contained lines outside the pgp signed "
706 ."part, cannot process\n" );
707 goto remove_only_changes;
708 } ## end if ( $extralines )
709 if ( !$main::mail_addr ) {
711 "$main::current_incoming_short/$changes doesn't contain a Maintainer: field; "
712 . "cannot process\n" );
713 goto remove_only_changes;
714 } ## end if ( !$main::mail_addr)
715 if ( $main::mail_addr !~ /^(buildd_\S+-\S+|\S+\@\S+\.\S+)/ ) {
717 # doesn't look like a mail address, maybe only the name
718 my ( $new_addr, @addr_list );
719 if ( $new_addr = try_to_get_mail_addr( $main::mail_addr, \@addr_list ) ) {
721 # substitute (unique) found addr, but give a warning
724 "(The Maintainer: field didn't contain a proper "
729 "Looking for `$main::mail_addr' in the Debian "
730 . "keyring gave your address\n"
732 msg( "mail", "as unique result, so I used this.)\n" );
734 "Substituted $new_addr for malformed " . "$main::mail_addr\n" );
735 $main::mail_addr = $new_addr;
738 # not found or not unique: hold the job and inform queue maintainer
739 my $old_addr = $main::mail_addr;
740 $main::mail_addr = $conf::maintainer_mail;
743 "The job ${main::current_incoming_short}/$changes doesn't have a correct email\n"
745 msg( "mail", "address in the Maintainer: field:\n" );
746 msg( "mail", " $old_addr\n" );
747 msg( "mail", "A check for this in the Debian keyring gave:\n" );
750 ? " " . join( ", ", @addr_list ) . "\n"
752 msg( "mail", "Please fix this manually\n" );
755 "Bad Maintainer: field in ${main::current_incoming_short}/$changes: $old_addr\n"
757 goto remove_only_changes;
758 } ## end else [ if ( $new_addr = try_to_get_mail_addr...
759 } ## end if ( $main::mail_addr ...
760 if ( $pgplines < 3 ) {
763 "$main::current_incoming_short/$changes isn't signed with PGP/GnuPG\n"
765 msg( "log", "(uploader $main::mail_addr)\n" );
766 goto remove_only_changes;
767 } ## end if ( $pgplines < 3 )
770 "$main::current_incoming_short/$changes doesn't mention any files\n" );
771 msg( "log", "(uploader $main::mail_addr)\n" );
772 goto remove_only_changes;
773 } ## end if ( !@files )
775 # check for packages that shouldn't be processed
776 if ( grep( $_ eq $pkgname, @conf::nonus_packages ) ) {
779 "$pkgname is a package that must be uploaded "
780 . "to nonus.debian.org\n"
782 msg( "log,mail", "instead of target.\n" );
784 "Job rejected and removed all files belonging " . "to it:\n" );
785 msg( "log,mail", " ", join( ", ", @filenames ), "\n" );
786 rm( $changes, @filenames );
788 } ## end if ( grep( $_ eq $pkgname...
790 $failure_file = $changes . ".failures";
791 $retries = $last_retry = 0;
792 if ( -f $failure_file ) {
793 open( FAILS, "<", $failure_file )
794 or die "Cannot open $main::current_incoming_short/$failure_file: $!\n";
797 ( $retries, $last_retry ) = ( $1, $2 )
798 if $line =~ /^(\d+)\s+(\d+)$/;
799 push( @$keep_list, $failure_file );
800 } ## end if ( -f $failure_file )
802 # run PGP on the file to check the signature
803 if ( !( $signator = pgp_check($changes) ) ) {
806 "$main::current_incoming_short/$changes has bad PGP/GnuPG signature!\n"
808 msg( "log", "(uploader $main::mail_addr)\n" );
812 "Removing $main::current_incoming_short/$changes, but keeping its associated ",
817 # Set SGID bit on associated files, so that the test for Debian files
818 # without a .changes doesn't consider them.
819 foreach (@filenames) {
821 next if !@st; # file may have disappeared in the meantime
822 chmod +( $st[ST_MODE] |= S_ISGID ), $_;
825 } elsif ( $signator eq "LOCAL ERROR" ) {
827 # An error has appened when starting pgp... Don't process the file,
828 # but also don't delete it
830 "Can't PGP/GnuPG check $main::current_incoming_short/$changes -- don't process it for now"
833 } ## end elsif ( $signator eq "LOCAL ERROR")
835 die "Cannot stat ${main::current_incoming_short}/$changes (??): $!\n"
836 if !( @changes_stats = stat($changes) );
838 # Make $upload_time the maximum of all modification times of files
839 # related to this .changes (and the .changes it self). This is the
840 # last time something changes to these files.
841 $upload_time = $changes_stats[ST_MTIME];
844 next if !( @stats = stat( $file->{"name"} ) );
845 $file->{"stats"} = \@stats;
846 $upload_time = $stats[ST_MTIME] if $stats[ST_MTIME] > $upload_time;
847 } ## end for $file (@files)
849 $do_report = ( time - $upload_time ) > $conf::problem_report_timeout;
850 $problems_reported = $changes_stats[ST_MODE] & S_ISGID;
852 # if any of the files is newer than the .changes' ctime (the time
853 # we sent a report and set the sticky bit), send new problem reports
854 if ( $problems_reported && $changes_stats[ST_CTIME] < $upload_time ) {
855 $problems_reported = 0;
856 chmod +( $changes_stats[ST_MODE] &= ~S_ISGID ), $changes;
857 debug("upload_time>changes-ctime => resetting problems reported");
859 debug("do_report=$do_report problems_reported=$problems_reported");
861 # now check all files for correct size and md5 sum
863 my $filename = $file->{"name"};
864 if ( !defined( $file->{"stats"} ) ) {
866 # could be an upload that isn't complete yet, be quiet,
867 # but don't process the file;
868 msg( "log,mail", "$filename doesn't exist\n" )
869 if $do_report && !$problems_reported;
870 msg( "log", "$filename doesn't exist (ignored for now)\n" )
872 msg( "log", "$filename doesn't exist (already reported)\n" )
873 if $problems_reported;
875 } elsif ( $file->{"stats"}->[ST_SIZE] < $file->{"size"}
879 # could be an upload that isn't complete yet, be quiet,
880 # but don't process the file
881 msg( "log", "$filename is too small (ignored for now)\n" );
883 } elsif ( $file->{"stats"}->[ST_SIZE] != $file->{"size"} ) {
884 msg( "log,mail", "$filename has incorrect size; deleting it\n" );
887 } elsif ( md5sum($filename) ne $file->{"md5"} ) {
889 "$filename has incorrect md5 checksum; ",
893 } ## end elsif ( md5sum($filename)...
894 } ## end for $file (@files)
897 if ( ( time - $upload_time ) > $conf::bad_changes_timeout ) {
899 # if a .changes fails for a really long time (several days
900 # or so), remove it and all associated files
903 "$main::current_incoming_short/$changes couldn't be processed for ",
904 int( $conf::bad_changes_timeout / ( 60 * 60 ) ),
905 " hours and is now deleted\n"
907 msg( "log,mail", "All files it mentions are also removed:\n" );
908 msg( "log,mail", " ", join( ", ", @filenames ), "\n" );
909 rm( $changes, @filenames, $failure_file );
910 } elsif ( $do_report && !$problems_reported ) {
912 # otherwise, send a problem report, if not done already
915 "Due to the errors above, the .changes file couldn't ",
917 "Please fix the problems for the upload to happen.\n"
920 # remember we already have sent a mail regarding this file
921 debug("Sending problem report mail and setting SGID bit");
922 my $mode = $changes_stats[ST_MODE] |= S_ISGID;
923 msg( "log", "chmod failed: $!" )
924 if ( chmod( $mode, $changes ) != 1 );
925 } ## end elsif ( $do_report && !$problems_reported)
932 # if this upload already failed earlier, wait until the delay requirement
935 && ( time - $last_retry ) <
936 ( $retries == 1 ? $conf::upload_delay_1 : $conf::upload_delay_2 ) )
938 msg( "log", "delaying retry of upload\n" );
940 } ## end if ( $retries > 0 && (...
942 if ( $conf::upload_method eq "ftp" ) {
943 return if !ftp_open();
946 # check if the job is already present on target
947 # (moved to here, to avoid bothering target as long as there are errors in
949 if ( $ls_l = is_on_target( $changes, @filenames ) ) {
952 "$main::current_incoming_short/$changes is already present on target host:\n"
954 msg( "log,mail", "$ls_l\n" );
956 "Either you already uploaded it, or someone else ",
958 msg( "log,mail", "Job $changes removed.\n" );
959 rm( $changes, @filenames, $failure_file );
961 } ## end if ( $ls_l = is_on_target...
963 # clear sgid bit before upload, scp would copy it to target. We don't need
964 # it anymore, we know there are no problems if we come here. Also change
965 # mode of files to 644 if this should be done locally.
966 $changes_stats[ST_MODE] &= ~S_ISGID;
967 if ( !$conf::chmod_on_target ) {
968 $changes_stats[ST_MODE] &= ~0777;
969 $changes_stats[ST_MODE] |= 0644;
971 chmod +( $changes_stats[ST_MODE] ), $changes;
973 # try uploading to target
974 if ( !copy_to_target( $changes, @filenames ) ) {
976 # if the upload failed, increment the retry counter and remember the
977 # current time; both things are written to the .failures file. Don't
978 # increment the fail counter if the error was due to incoming
980 return if !$main::incoming_writable;
981 if ( ++$retries >= $conf::max_upload_retries ) {
983 "$changes couldn't be uploaded for $retries times now.\n" );
985 "Giving up and removing it and its associated files:\n" );
986 msg( "log,mail", " ", join( ", ", @filenames ), "\n" );
987 rm( $changes, @filenames, $failure_file );
990 if ( open( FAILS, ">", $failure_file ) ) {
991 print FAILS "$retries $last_retry\n";
993 chmod( 0600, $failure_file )
994 or die "Cannot set modes of $failure_file: $!\n";
995 } ## end if ( open( FAILS, ">$failure_file"...
996 push( @$keep_list, $failure_file );
997 debug("now $retries failed uploads");
1000 "The upload will be retried in ",
1003 ? $conf::upload_delay_1
1004 : $conf::upload_delay_2
1008 } ## end else [ if ( ++$retries >= $conf::max_upload_retries)
1010 } ## end if ( !copy_to_target( ...
1012 # If the files were uploaded ok, remove them
1013 rm( $changes, @filenames, $failure_file );
1015 msg( "mail", "$changes uploaded successfully to $conf::target\n" );
1016 msg( "mail", "along with the files:\n ", join( "\n ", @filenames ),
1019 "$changes processed successfully (uploader $main::mail_addr)\n" );
1021 # Check for files that have the same stem as the .changes (and weren't
1022 # mentioned there) and delete them. It happens often enough that people
1023 # upload a .orig.tar.gz where it isn't needed and also not in the
1024 # .changes. Explicitly deleting it (and not waiting for the
1025 # $stray_remove_timeout) reduces clutter in the queue dir and maybe also
1026 # educates uploaders :-)
1028 # my $pattern = debian_file_stem( $changes );
1029 # my $spattern = substr( $pattern, 0, -1 ); # strip off '*' at end
1030 # my @other_files = glob($pattern);
1031 # filter out files that have a Debian revision at all and a different
1032 # revision. Those belong to a different upload.
1033 # if ($changes =~ /^\Q$spattern\E-([\d.+-]+)/) {
1034 # my $this_rev = $1;
1035 # @other_files = grep( !/^\Q$spattern\E-([\d.+-]+)/ || $1 eq $this_rev,
1038 # Also do not remove those files if a .changes is among them. Then there
1039 # is probably a second upload for another version or another architecture.
1040 # if (@other_files && !grep( /\.changes$/, @other_files )) {
1041 # rm( @other_files );
1042 # msg( "mail", "\nThe following file(s) seemed to belong to the same ".
1043 # "upload, but weren't listed\n" );
1044 # msg( "mail", "in the .changes file:\n " );
1045 # msg( "mail", join( "\n ", @other_files ), "\n" );
1046 # msg( "mail", "They have been deleted.\n" );
1047 # msg( "log", "Deleted files in upload not in $changes: @other_files\n" );
1049 } ## end sub process_changes($\@)
1052 # process one .dak-commands file
1054 sub process_dak_commands {
1055 my $commands = shift;
1057 msg("log", "processing ${main::current_incoming_short}/$commands\n");
1059 # TODO: get mail address from signed contents
1060 # and NOT implement a third parser for armored PGP...
1061 $main::mail_addr = undef;
1064 my $signator = pgp_check($commands);
1067 "$main::current_incoming_short/$commands has bad PGP/GnuPG signature!\n");
1069 "Removing $main::current_incoming_short/$commands\n");
1073 elsif ($signator eq 'LOCAL ERROR') {
1074 debug("Can't check signature for $main::current_incoming_short/$commands -- don't process it for now");
1077 msg("log,mail", "(PGP/GnuPG signature by $signator)\n");
1080 my @filenames = ($commands);
1081 if (my $ls_l = is_on_target($commands, @filenames)) {
1082 msg("log,mail", "$main::current_incoming_short/$commands is already present on target host:\n");
1083 msg("log,mail", "$ls_l\n");
1084 msg("log,mail", "Job $commands removed.\n");
1089 if (!copy_to_target($commands)) {
1090 msg("log,mail", "$commands couldn't be uploaded to target.\n");
1091 msg("log,mail", "Giving up and removing it.\n");
1097 msg("mail", "$commands uploaded successfully to $conf::target\n");
1101 # process one .commands file
1103 sub process_commands($) {
1104 my $commands = shift;
1105 my ( @cmds, $cmd, $pgplines, $signator );
1107 my ( @files, $file, @removed, $target_delay );
1109 format_status_str( $main::current_changes, $commands );
1111 write_status_file() if $conf::statusdelay;
1113 msg( "log", "processing $main::current_incoming_short/$commands\n" );
1115 # parse the .commands file
1116 if ( !open( COMMANDS, "<", $commands ) ) {
1117 msg( "log", "Cannot open $main::current_incoming_short/$commands: $!\n" );
1121 $main::mail_addr = "";
1123 outer_loop: while (<COMMANDS>) {
1124 if (/^---+(BEGIN|END) PGP .*---+$/) {
1126 } elsif (/^Uploader:\s*/i) {
1127 chomp( $main::mail_addr = $' );
1128 $main::mail_addr = $1 if $main::mail_addr =~ /<([^>]*)>/;
1129 } elsif (/^Commands:/i) {
1132 s/^\s*(.*)\s*$/$1/; # delete whitespace at both ends
1135 debug("includes cmd $_");
1137 last outer_loop if !defined( $_ = scalar(<COMMANDS>) );
1139 redo outer_loop if !/^\s/ || /^$/;
1140 } ## end for ( ; ; )
1141 } ## end elsif (/^Commands:/i)
1142 } ## end while (<COMMANDS>)
1145 # some consistency checks
1146 if ( !$main::mail_addr || $main::mail_addr !~ /^\S+\@\S+\.\S+/ ) {
1148 "$main::current_incoming_short/$commands contains no or bad Uploader: field: "
1149 . "$main::mail_addr\n" );
1151 "cannot process $main::current_incoming_short/$commands\n" );
1152 $main::mail_addr = "";
1154 } ## end if ( !$main::mail_addr...
1155 msg( "log", "(command uploader $main::mail_addr)\n" );
1157 if ( $pgplines < 3 ) {
1160 "$main::current_incoming_short/$commands isn't signed with PGP/GnuPG\n"
1164 "or the uploaded file is broken. Make sure to transfer in binary mode\n"
1166 msg( "mail", "or better yet - use dcut for commands files\n" );
1168 } ## end if ( $pgplines < 3 )
1170 # run PGP on the file to check the signature
1171 if ( !( $signator = pgp_check($commands) ) ) {
1174 "$main::current_incoming_short/$commands has bad PGP/GnuPG signature!\n"
1177 msg( "log,mail", "Removing $main::current_incoming_short/$commands\n" );
1180 } elsif ( $signator eq "LOCAL ERROR" ) {
1182 # An error has appened when starting pgp... Don't process the file,
1183 # but also don't delete it
1185 "Can't PGP/GnuPG check $main::current_incoming_short/$commands -- don't process it for now"
1188 } ## end elsif ( $signator eq "LOCAL ERROR")
1189 msg( "log", "(PGP/GnuPG signature by $signator)\n" );
1191 # now process commands
1194 "Log of processing your commands file $main::current_incoming_short/$commands:\n\n"
1196 foreach $cmd (@cmds) {
1197 my @word = split( /\s+/, $cmd );
1198 msg( "mail,log", "> @word\n" );
1199 my $selecteddelayed = -1;
1202 if ( $word[0] eq "rm" ) {
1203 foreach ( @word[ 1 .. $#word ] ) {
1205 if (m,^DELAYED/([0-9]+)-day/,) {
1206 $selecteddelayed = $1;
1207 s,^DELAYED/[0-9]+-day/,,;
1210 msg("mail,log", "$_: filename component cannot start with a wildcard\n");
1211 } elsif ( $origword eq "--searchdirs" ) {
1212 $selecteddelayed = -2;
1216 "$_: filename may not contain slashes except for DELAYED/#-day/ prefixes\n"
1220 # process wildcards but also plain names
1222 my $pat = quotemeta($_);
1223 $pat =~ s/\\\*/.*/g;
1224 $pat =~ s/\\\?/.?/g;
1225 $pat =~ s/\\([][])/$1/g;
1227 if ( $selecteddelayed < 0 ) { # scanning or explicitly incoming
1228 opendir( DIR, "." );
1229 push( @thesefiles, grep /^$pat$/, readdir(DIR) );
1232 if ( $selecteddelayed >= 0 ) {
1233 my $dir = sprintf( $conf::incoming_delayed, $selecteddelayed );
1234 opendir( DIR, $dir );
1236 map ( "$dir/$_", grep /^$pat$/, readdir(DIR) ) );
1238 } elsif ( $selecteddelayed == -2 ) {
1239 for ( my ($adelay) = 0 ;
1240 ( !@thesefiles ) && $adelay <= $conf::max_delayed ;
1243 my $dir = sprintf( $conf::incoming_delayed, $adelay );
1244 opendir( DIR, $dir );
1246 map ( "$dir/$_", grep /^$pat$/, readdir(DIR) ) );
1248 } ## end for ( my ($adelay) = 0 ...
1249 } ## end elsif ( $selecteddelayed ...
1250 push( @files, @thesefiles );
1251 if ( !@thesefiles ) {
1252 msg( "mail,log", "$origword did not match anything\n" );
1254 } ## end else [ if ( $origword eq "--searchdirs")
1255 } ## end foreach ( @word[ 1 .. $#word...
1257 msg( "mail,log", "No files to delete\n" );
1260 foreach $file (@files) {
1262 msg( "mail,log", "$file: no such file\n" );
1263 } elsif ( $file =~ /$conf::keep_files/ ) {
1264 msg( "mail,log", "$file is protected, cannot " . "remove\n" );
1265 } elsif ( !unlink($file) ) {
1266 msg( "mail,log", "$file: rm: $!\n" );
1268 $file =~ s,$conf::incoming/?,,;
1269 push( @removed, $file );
1271 } ## end foreach $file (@files)
1272 msg( "mail,log", "Files removed: @removed\n" ) if @removed;
1273 } ## end else [ if ( !@files )
1274 } elsif ( $word[0] eq "reschedule" ) {
1276 msg( "mail,log", "Wrong number of arguments\n" );
1277 } elsif ( $conf::upload_method ne "copy" ) {
1278 msg( "mail,log", "reschedule not available\n" );
1279 } elsif ( $word[1] =~ m,/, || $word[1] !~ m/\.changes/ ) {
1282 "$word[1]: filename may not contain slashes and must be .changes\n"
1284 } elsif ( !( ($target_delay) = $word[2] =~ m,^([0-9]+)-day$, )
1285 || $target_delay > $conf::max_delayed )
1289 "$word[2]: rescheduling target must be #-day with # between 0 and $conf::max_delayed (in particular, no '/' allowed)\n"
1291 } elsif ( $word[1] =~ /$conf::keep_files/ ) {
1292 msg( "mail,log", "$word[1] is protected, cannot do stuff with it\n" );
1296 $adelay <= $conf::max_delayed
1298 sprintf( "$conf::targetdir_delayed", $adelay ) . "/$word[1]" ) ;
1301 } ## end for ( $adelay = 0 ; $adelay...
1302 if ( $adelay > $conf::max_delayed ) {
1303 msg( "mail,log", "$word[1] not found\n" );
1304 } elsif ( $adelay == $target_delay ) {
1305 msg( "mail,log", "$word[1] already is in $word[2]\n" );
1308 my ($dir) = sprintf( "$conf::targetdir_delayed", $adelay );
1310 sprintf( "$conf::targetdir_delayed", $target_delay );
1311 push( @thesefiles, $word[1] );
1313 get_filelist_from_known_good_changes("$dir/$word[1]") );
1314 for my $afile (@thesefiles) {
1315 if ( $afile =~ m/\.changes$/ ) {
1316 utime undef, undef, ("$dir/$afile");
1318 if ( !move("$dir/$afile", "$target_dir/$afile") ) {
1319 msg( "mail,log", "move: $!\n" );
1321 msg( "mail,log", "$afile moved to $target_delay-day\n" );
1323 } ## end for my $afile (@thesefiles)
1324 } ## end else [ if ( $adelay > $conf::max_delayed)
1325 } ## end else [ if ( @word != 3 )
1326 } elsif ( $word[0] eq "cancel" ) {
1328 msg( "mail,log", "Wrong number of arguments\n" );
1329 } elsif ( $conf::upload_method ne "copy" ) {
1330 msg( "mail,log", "cancel not available\n" );
1332 $word[1] !~ m,$re_file_safe_prefix.changes\z, )
1335 "argument to cancel must be one .changes filename without path\n" );
1336 } ## end elsif ( $word[1] !~ ...
1338 for ( my ($adelay) = 0 ; $adelay <= $conf::max_delayed ; $adelay++ ) {
1339 my ($dir) = sprintf( "$conf::targetdir_delayed", $adelay );
1340 if ( -f "$dir/$word[1]" ) {
1342 push( @files, "$word[1]" );
1344 get_filelist_from_known_good_changes("$dir/$word[1]") );
1345 foreach $file (@files) {
1346 if ( !-f "$dir/$file" ) {
1347 msg( "mail,log", "$dir/$file: no such file\n" );
1348 } elsif ( "$dir/$file" =~ /$conf::keep_files/ ) {
1350 "$dir/$file is protected, cannot " . "remove\n" );
1351 } elsif ( !unlink("$dir/$file") ) {
1352 msg( "mail,log", "$dir/$file: rm: $!\n" );
1354 push( @removed, $file );
1356 } ## end foreach $file (@files)
1357 msg( "mail,log", "Files removed from $adelay-day: @removed\n" )
1359 } ## end if ( -f "$dir/$word[1]")
1360 } ## end for ( my ($adelay) = 0 ...
1362 msg( "mail,log", "No upload found: $word[1]\n" );
1365 msg( "mail,log", "unknown command $word[0]\n" );
1367 } ## end foreach $cmd (@cmds)
1370 "-- End of $main::current_incoming_short/$commands processing\n" );
1371 } ## end sub process_commands($)
1373 sub age_delayed_queues() {
1374 for ( my ($adelay) = 0 ; $adelay <= $conf::max_delayed ; $adelay++ ) {
1375 my ($dir) = sprintf( "$conf::targetdir_delayed", $adelay );
1377 if ( $adelay == 0 ) {
1378 $target_dir = $conf::targetdir;
1380 $target_dir = sprintf( "$conf::targetdir_delayed", $adelay - 1 );
1382 for my $achanges (<$dir/*.changes>) {
1383 my $mtime = ( stat($achanges) )[9];
1384 if ( $mtime + 24 * 60 * 60 <= time || $adelay == 0 ) {
1385 utime undef, undef, ($achanges);
1386 my @thesefiles = ( $achanges =~ m,.*/([^/]*), );
1387 push( @thesefiles, get_filelist_from_known_good_changes($achanges) );
1388 for my $afile (@thesefiles) {
1389 if ( !move("$dir/$afile", "$target_dir/$afile") ) {
1390 msg( "log", "move: $!\n" );
1392 msg( "log", "$afile moved to $target_dir\n" );
1394 } ## end for my $afile (@thesefiles)
1395 } ## end if ( $mtime + 24 * 60 ...
1396 } ## end for my $achanges (<$dir/*.changes>)
1397 } ## end for ( my ($adelay) = 0 ...
1398 } ## end sub age_delayed_queues()
1401 # check if a file is already on target
1403 sub is_on_target($\@) {
1405 my $filelist = shift;
1409 if ( $conf::upload_method eq "ssh" ) {
1410 ( $msg, $stat ) = ssh_cmd("ls -l $file");
1411 } elsif ( $conf::upload_method eq "ftp" ) {
1413 ( $msg, $err ) = ftp_cmd( "dir", $file );
1419 $msg = "ls: no such file\n";
1422 $msg = join( "\n", @$msg );
1425 my @allfiles = ($file);
1426 push( @allfiles, @$filelist );
1428 $msg = "no such file";
1429 for my $afile (@allfiles) {
1430 if ( -f "$conf::targetdir/$afile" ) {
1434 } ## end for my $afile (@allfiles)
1435 for ( my ($adelay) = 0 ;
1436 $adelay <= $conf::max_delayed && $stat ;
1439 for my $afile (@allfiles) {
1441 -f ( sprintf( "$conf::targetdir_delayed", $adelay ) . "/$afile" ) )
1444 $msg = sprintf( "%d-day", $adelay ) . "/$afile";
1445 } ## end if ( -f ( sprintf( "$conf::targetdir_delayed"...
1446 } ## end for my $afile (@allfiles)
1447 } ## end for ( my ($adelay) = 0 ...
1448 } ## end else [ if ( $conf::upload_method...
1450 debug("exit status: $stat, output was: $msg");
1452 return "" if $stat && $msg =~ /no such file/i; # file not present
1453 msg( "log", "strange ls -l output on target:\n", $msg ), return ""
1454 if $stat || $@; # some other error, but still try to upload
1456 # ls -l returned 0 -> file already there
1457 $msg =~ s/\s\s+/ /g; # make multiple spaces into one, to save space
1459 } ## end sub is_on_target($\@)
1462 # copy a list of files to target
1464 sub copy_to_target(@) {
1466 my ( @md5sum, @expected_files, $sum, $name, $msgs, $stat );
1469 write_status_file() if $conf::statusdelay;
1472 if ( $conf::upload_method eq "ssh" ) {
1473 ( $msgs, $stat ) = scp_cmd(@files);
1475 } elsif ( $conf::upload_method eq "ftp" ) {
1477 if ( !$main::FTP_chan->cwd($main::current_targetdir) ) {
1479 "Can't cd to $main::current_targetdir on $conf::target\n" );
1482 foreach $file (@files) {
1483 ( $rv, $msgs ) = ftp_cmd( "put", $file );
1488 local_cmd( "$conf::cp @files $main::current_targetdir", 'NOCD' );
1492 # check md5sums or sizes on target against our own
1493 my $have_md5sums = 1;
1494 if ($conf::check_md5sum) {
1495 if ( $conf::upload_method eq "ssh" ) {
1496 ( $msgs, $stat ) = ssh_cmd("md5sum @files");
1498 @md5sum = split( "\n", $msgs );
1499 } elsif ( $conf::upload_method eq "ftp" ) {
1500 my ( $rv, $err, $file );
1501 foreach $file (@files) {
1502 ( $rv, $err ) = ftp_cmd( "quot", "site", "md5sum", $file );
1504 next if ftp_code() == 550; # file not found
1505 if ( ftp_code() == 500 ) { # unimplemented
1507 goto get_sizes_instead;
1512 chomp( my $t = ftp_response() );
1513 push( @md5sum, $t );
1514 } ## end foreach $file (@files)
1515 if ( !$have_md5sums ) {
1517 foreach $file (@files) {
1518 ( $rv, $err ) = ftp_cmd( "size", $file );
1520 next if ftp_code() == 550; # file not found
1524 push( @md5sum, "$rv $file" );
1525 } ## end foreach $file (@files)
1526 } ## end if ( !$have_md5sums )
1528 ( $msgs, $stat ) = local_cmd("$conf::md5sum @files");
1530 @md5sum = split( "\n", $msgs );
1533 @expected_files = @files;
1536 ( $sum, $name ) = split;
1537 next if !grep { $_ eq $name } @files; # a file we didn't upload??
1538 next if $sum eq "md5sum:"; # looks like an error message
1539 if ( ( $have_md5sums && $sum ne md5sum($name) )
1540 || ( !$have_md5sums && $sum != ( -s $name ) ) )
1544 "Upload of $name to $conf::target failed ",
1545 "(" . ( $have_md5sums ? "md5sum" : "size" ) . " mismatch)\n"
1548 } ## end if ( ( $have_md5sums &&...
1550 # seen that file, remove it from expect list
1551 @expected_files = map { $_ eq $name ? () : $_ } @expected_files;
1552 } ## end foreach (@md5sum)
1553 if (@expected_files) {
1554 msg( "log,mail", "Failed to upload the files\n" );
1555 msg( "log,mail", " ", join( ", ", @expected_files ), "\n" );
1556 msg( "log,mail", "(Not present on target after upload)\n" );
1558 } ## end if (@expected_files)
1559 } ## end if ($conf::check_md5sum)
1561 if ($conf::chmod_on_target) {
1563 # change file's mode explicitly to 644 on target
1564 if ( $conf::upload_method eq "ssh" ) {
1565 ( $msgs, $stat ) = ssh_cmd("chmod 644 @files");
1567 } elsif ( $conf::upload_method eq "ftp" ) {
1569 foreach $file (@files) {
1570 ( $rv, $msgs ) = ftp_cmd( "quot", "site", "chmod", "644", $file );
1571 msg( "log", "Can't chmod $file on target:\n$msgs" )
1574 } ## end foreach $file (@files)
1576 ( $msgs, $stat ) = local_cmd("$conf::chmod 644 @files");
1579 } ## end if ($conf::chmod_on_target)
1582 write_status_file() if $conf::statusdelay;
1587 "Upload to $conf::target failed",
1588 $? ? ", last exit status " . sprintf( "%s", $? >> 8 ) : "", "\n" );
1589 msg( "log,mail", "Error messages:\n", $msgs )
1592 # If "permission denied" was among the errors, test if the incoming is
1594 if ( $msgs =~ /(permission denied|read-?only file)/i ) {
1595 if ( !check_incoming_writable() ) {
1596 msg( "log,mail", "(The incoming directory seems to be ",
1599 } ## end if ( $msgs =~ /(permission denied|read-?only file)/i)
1601 # remove bad files or an incomplete upload on target
1602 if ( $conf::upload_method eq "ssh" ) {
1603 ssh_cmd("rm -f @files");
1604 } elsif ( $conf::upload_method eq "ftp" ) {
1606 foreach $file (@files) {
1608 ( $rv, $err ) = ftp_cmd( "delete", $file );
1609 msg( "log", "Can't delete $file on target:\n$err" )
1611 } ## end foreach $file (@files)
1613 my @tfiles = map { "$main::current_targetdir/$_" } @files;
1614 debug("executing unlink(@tfiles)");
1618 write_status_file() if $conf::statusdelay;
1620 } ## end sub copy_to_target(@)
1623 # check if a file is correctly signed with PGP
1633 if ($file =~ /$re_file_safe/) {
1636 msg( "log", "Tainted filename, skipping: $file\n" );
1637 return "LOCAL ERROR";
1641 if ( -x $conf::gpg ) {
1642 debug( "executing $conf::gpg --no-options --batch "
1643 . "--no-default-keyring --always-trust "
1645 . join( " --keyring ", @conf::keyrings )
1646 . " --verify '$file'" );
1649 "$conf::gpg --no-options --batch "
1650 . "--no-default-keyring --always-trust "
1652 . join( " --keyring ", @conf::keyrings )
1653 . " --verify '$file'"
1658 msg( "log", "Can't open pipe to $conf::gpg: $!\n" );
1659 return "LOCAL ERROR";
1660 } ## end if ( !open( PIPE, "$conf::gpg --no-options --batch "...
1661 $output .= $_ while (<PIPE>);
1664 } ## end if ( -x $conf::gpg )
1667 msg( "log,mail", "GnuPG signature check failed on $file\n" );
1668 msg( "mail", $output );
1669 msg( "log,mail", "(Exit status ", $stat >> 8, ")\n" );
1673 $output =~ /^(gpg: )?good signature from (user )?"(.*)"\.?$/im;
1674 ( $signator = $3 ) ||= "unknown signator";
1676 debug("GnuPG signature ok (by $signator)");
1679 } ## end sub pgp_check($)
1681 # ---------------------------------------------------------------------------
1683 # ---------------------------------------------------------------------------
1686 # fork a subprocess that watches the 'status' FIFO
1688 # that process blocks until someone opens the FIFO, then sends a
1689 # signal (SIGUSR1) to the main process, expects
1691 sub fork_statusd() {
1697 $statusd_pid = open( STATUSD, "|-" );
1698 die "cannot fork: $!\n" if !defined($statusd_pid);
1700 # parent just returns
1702 msg( "log", "forked status daemon (pid $statusd_pid)\n" );
1703 return $statusd_pid;
1706 # child: the status FIFO daemon
1708 # ignore SIGPIPE here, in case some closes the FIFO without completely
1710 $SIG{"PIPE"} = "IGNORE";
1712 # also ignore SIGCLD, we don't want to inherit the restart-statusd handler
1714 $SIG{"CHLD"} = "DEFAULT";
1716 rm($conf::statusfile);
1717 $errs = `$conf::mkfifo $conf::statusfile`;
1718 die "$main::progname: cannot create named pipe $conf::statusfile: $errs"
1720 chmod( 0644, $conf::statusfile )
1721 or die "Cannot set modes of $conf::statusfile: $!\n";
1723 # close log file, so that log rotating works
1729 my ( $status, $mup, $incw, $ds, $next_run, $last_ping, $currch, $l );
1731 # open the FIFO for writing; this blocks until someone (probably ftpd)
1732 # opens it for reading
1733 open( STATFIFO, ">", $conf::statusfile )
1734 or die "Cannot open $conf::statusfile\n";
1737 # tell main daemon to send us status infos
1738 kill( $main::signo{"USR1"}, $main_pid );
1740 # get the infos from stdin; must loop until enough bytes received!
1741 my $expect_len = 3 + 2 * STATNUM_LEN + STATSTR_LEN;
1742 for ( $status = "" ; ( $l = length($status) ) < $expect_len ; ) {
1743 sysread( STDIN, $status, $expect_len - $l, $l );
1746 # disassemble the status byte stream
1752 [ next_run => STATNUM_LEN ],
1753 [ last_ping => STATNUM_LEN ],
1754 [ currch => STATSTR_LEN ]
1757 eval "\$$_->[0] = substr( \$status, $pos, $_->[1] );";
1759 } ## end foreach ( [ mup => 1 ], [ incw...
1760 $currch =~ s/\n+//g;
1762 print_status( $mup, $incw, $ds, $next_run, $last_ping, $currch );
1765 # This sleep is necessary so that we can't reopen the FIFO
1766 # immediately, in case the reader hasn't closed it yet if we get to
1767 # the open again. Is there a better solution for this??
1770 } ## end sub fork_statusd()
1773 # update the status file, in case we use a plain file and not a FIFO
1775 sub write_status_file() {
1777 return if !$conf::statusfile;
1779 open( STATFILE, ">", $conf::statusfile )
1780 or ( msg( "log", "Could not open $conf::statusfile: $!\n" ), return );
1781 my $oldsel = select(STATFILE);
1784 $main::target_up, $main::incoming_writable,
1785 $main::dstat, $main::next_run,
1786 $main::last_ping_time, $main::current_changes
1791 } ## end sub write_status_file()
1793 sub print_status($$$$$$) {
1797 my $next_run = shift;
1798 my $last_ping = shift;
1803 ( $version = 'Release: 0.9 $Revision: 1.51 $' ) =~ s/\$ ?//g;
1804 print "debianqueued $version\n";
1806 $approx = $conf::statusdelay ? "approx. " : "";
1808 if ( $mup eq "0" ) {
1809 print "$conf::target is down, queue pausing\n";
1811 } elsif ( $conf::upload_method ne "copy" ) {
1812 print "$conf::target seems to be up, last ping $approx",
1813 print_time( time - $last_ping ), " ago\n";
1816 if ( $incw eq "0" ) {
1817 print "The incoming directory is not writable, queue pausing\n";
1822 print "Next queue check in $approx", print_time( $next_run - time ), "\n";
1824 } elsif ( $ds eq "c" ) {
1825 print "Checking queue directory\n";
1826 } elsif ( $ds eq "u" ) {
1827 print "Uploading to $conf::target\n";
1829 print "Bad status data from daemon: \"$mup$incw$ds\"\n";
1833 print "Current job is $currch\n" if $currch;
1834 } ## end sub print_status($$$$$$)
1837 # format a number for sending to statusd (fixed length STATNUM_LEN)
1839 sub format_status_num(\$$) {
1843 $$varref = sprintf "%" . STATNUM_LEN . "d", $num;
1844 } ## end sub format_status_num(\$$)
1847 # format a string for sending to statusd (fixed length STATSTR_LEN)
1849 sub format_status_str(\$$) {
1853 $$varref = substr( $str, 0, STATSTR_LEN );
1854 $$varref .= "\n" x ( STATSTR_LEN - length($$varref) );
1855 } ## end sub format_status_str(\$$)
1858 # send a status string to the status daemon
1860 # Avoid all operations that could call malloc() here! Most libc
1861 # implementations aren't reentrant, so we may not call it from a
1862 # signal handler. So use only already-defined variables.
1865 local $! = 0; # preserve errno
1867 # re-setup handler, in case we have broken SysV signals
1868 $SIG{"USR1"} = \&send_status;
1870 syswrite( STATUSD, $main::target_up, 1 );
1871 syswrite( STATUSD, $main::incoming_writable, 1 );
1872 syswrite( STATUSD, $main::dstat, 1 );
1873 syswrite( STATUSD, $main::next_run, STATNUM_LEN );
1874 syswrite( STATUSD, $main::last_ping_time, STATNUM_LEN );
1875 syswrite( STATUSD, $main::current_changes, STATSTR_LEN );
1876 } ## end sub send_status()
1878 # ---------------------------------------------------------------------------
1880 # ---------------------------------------------------------------------------
1883 # open FTP connection to target host if not already open
1887 if ($main::FTP_chan) {
1889 # is already open, but might have timed out; test with a cwd
1890 return $main::FTP_chan
1891 if $main::FTP_chan->cwd($main::current_targetdir);
1893 # cwd didn't work, channel is closed, try to reopen it
1894 $main::FTP_chan = undef;
1895 } ## end if ($main::FTP_chan)
1902 Debug => $conf::ftpdebug,
1903 Timeout => $conf::ftptimeout,
1909 msg( "log,mail", "Cannot open FTP server $conf::target\n" );
1911 } ## end if ( !( $main::FTP_chan...
1912 if ( !$main::FTP_chan->login() ) {
1913 msg( "log,mail", "Anonymous login on FTP server $conf::target failed\n" );
1916 if ( !$main::FTP_chan->binary() ) {
1917 msg( "log,mail", "Can't set binary FTP mode on $conf::target\n" );
1920 if ( !$main::FTP_chan->cwd($main::current_targetdir) ) {
1922 "Can't cd to $main::current_targetdir on $conf::target\n" );
1925 debug("opened FTP channel to $conf::target");
1929 $main::FTP_chan = undef;
1931 } ## end sub ftp_open()
1936 my $direct_resp_cmd = ( $cmd eq "quot" );
1938 debug( "executing FTP::$cmd(" . join( ", ", @_ ) . ")" );
1939 $SIG{"ALRM"} = sub { die "timeout in FTP::$cmd\n" };
1940 alarm($conf::remote_timeout);
1941 eval { $rv = $main::FTP_chan->$cmd(@_); };
1944 $rv = ( ftp_code() =~ /^2/ ) ? 1 : 0 if $direct_resp_cmd;
1949 $err = ftp_response();
1951 return ( $rv, $err );
1952 } ## end sub ftp_cmd($@)
1955 if ($main::FTP_chan) {
1956 $main::FTP_chan->quit();
1957 $main::FTP_chan = undef;
1960 } ## end sub ftp_close()
1962 sub ftp_response() {
1963 return join( '', @{ ${*$main::FTP_chan}{'net_cmd_resp'} } );
1967 return ${*$main::FTP_chan}{'net_cmd_code'};
1971 my $code = ftp_code();
1972 return ( $code =~ /^[45]/ ) ? 1 : 0;
1975 # ---------------------------------------------------------------------------
1977 # ---------------------------------------------------------------------------
1983 my $ecmd = "$conf::ssh $conf::ssh_options $conf::target "
1984 . "-l $conf::targetlogin \'cd $main::current_targetdir; $cmd\'";
1985 debug("executing $ecmd");
1986 $SIG{"ALRM"} = sub { die "timeout in ssh command\n" };
1987 alarm($conf::remote_timeout);
1988 eval { $msg = `$ecmd 2>&1`; };
1996 return ( $msg, $stat );
1997 } ## end sub ssh_cmd($)
2002 my $ecmd = "$conf::scp $conf::ssh_options @_ "
2003 . "$conf::targetlogin\@$conf::target:$main::current_targetdir";
2004 debug("executing $ecmd");
2005 $SIG{"ALRM"} = sub { die "timeout in scp\n" };
2006 alarm($conf::remote_timeout);
2007 eval { $msg = `$ecmd 2>&1`; };
2015 return ( $msg, $stat );
2016 } ## end sub scp_cmd(@)
2018 sub local_cmd($;$) {
2023 my $ecmd = ( $nocd ? "" : "cd $main::current_targetdir; " ) . $cmd;
2024 debug("executing $ecmd");
2025 $msg = `($ecmd) 2>&1`;
2027 return ( $msg, $stat );
2029 } ## end sub local_cmd($;$)
2032 # check if target is alive (code stolen from Net::Ping.pm)
2034 sub check_alive(;$) {
2035 my $timeout = shift;
2036 my ( $saddr, $ret, $target_ip );
2039 if ( $conf::upload_method eq "copy" ) {
2040 format_status_num( $main::last_ping_time, time );
2041 $main::target_up = 1;
2047 if ( !( $target_ip = ( gethostbyname($conf::target) )[4] ) ) {
2048 msg( "log", "Cannot get IP address of $conf::target\n" );
2052 $saddr = pack( 'S n a4 x8', AF_INET, $main::echo_port, $target_ip );
2053 $SIG{'ALRM'} = sub { die };
2056 $ret = $main::tcp_proto; # avoid warnings about unused variable
2059 return unless socket( PINGSOCK, PF_INET, SOCK_STREAM, $main::tcp_proto );
2060 return unless connect( PINGSOCK, $saddr );
2065 msg( "log", "pinging $conf::target: " . ( $ret ? "ok" : "down" ) . "\n" );
2067 $main::target_up = $ret ? "1" : "0";
2068 format_status_num( $main::last_ping_time, time );
2069 write_status_file() if $conf::statusdelay;
2070 } ## end sub check_alive(;$)
2073 # check if incoming dir on target is writable
2075 sub check_incoming_writable() {
2076 my $testfile = ".debianqueued-testfile";
2079 if ( $conf::upload_method eq "ssh" ) {
2081 ssh_cmd( "rm -f $testfile; touch $testfile; " . "rm -f $testfile" );
2082 } elsif ( $conf::upload_method eq "ftp" ) {
2083 my $file = "junk-for-writable-test-" . format_time();
2084 $file =~ s/[ :.]/-/g;
2086 open( F, ">", $file );
2089 ( $rv, $msg ) = ftp_cmd( "put", $file );
2091 $msg = "" if !defined $msg;
2093 ftp_cmd( "delete", $file );
2094 } elsif ( $conf::upload_method eq "copy" ) {
2096 local_cmd( "rm -f $testfile; touch $testfile; " . "rm -f $testfile" );
2099 debug("exit status: $stat, output was: $msg");
2103 # change incoming_writable only if ssh didn't return an error
2104 $main::incoming_writable =
2105 ( $msg =~ /(permission denied|read-?only file|cannot create)/i )
2109 debug("local error, keeping old status");
2111 debug("incoming_writable = $main::incoming_writable");
2112 write_status_file() if $conf::statusdelay;
2113 return $main::incoming_writable;
2114 } ## end sub check_incoming_writable()
2117 # remove a list of files, log failing ones
2123 ( unlink $_ and ++$done )
2125 or msg( "log", "Could not delete $_: $!\n" );
2131 # get md5 checksum of a file
2136 open my $fh, "<", $file or return "";
2137 my $md5 = $md5->addfile($fh);
2140 return $md5->hexdigest;
2141 } ## end sub md5sum($)
2144 # check if a file probably belongs to a Debian upload
2146 sub is_debian_file($) {
2148 return $file =~ /\.(deb|dsc|(diff|tar)\.gz)$/
2149 && $file !~ /\.orig\.tar\.gz/;
2153 # return a pattern that matches all files that probably belong to one job
2155 sub debian_file_stem($) {
2157 my ( $pkg, $version );
2160 $file =~ s,\.(deb|dsc|changes|(orig\.)?tar\.gz|diff\.gz)$,,;
2162 # if not is *_* (name_version), can't derive a stem and return just
2164 return $file if !( $file =~ /^([^_]+)_([^_]+)/ );
2165 ( $pkg, $version ) = ( $1, $2 );
2167 # strip Debian revision from version
2168 $version =~ s/^(.*)-[\d.+-]+$/$1/;
2170 return "${pkg}_${version}*";
2171 } ## end sub debian_file_stem($)
2174 # output a messages to several destinations
2176 # first arg is a comma-separated list of destinations; valid are "log"
2177 # and "mail"; rest is stuff to be printed, just as with print
2180 my @dest = split( ',', shift );
2182 if ( grep /log/, @dest ) {
2183 my $now = format_time();
2184 print LOG "$now ", @_;
2187 if ( grep /mail/, @dest ) {
2188 $main::mail_text .= join( '', @_ );
2190 } ## end sub msg($@)
2193 # print a debug messages, if $debug is true
2196 return if !$conf::debug;
2197 my $now = format_time();
2198 print LOG "$now DEBUG ", @_, "\n";
2202 # intialize the "mail" destination of msg() (this clears text,
2203 # address, subject, ...)
2208 $main::mail_addr = "";
2209 $main::mail_text = "";
2210 %main::packages = ();
2211 $main::mail_subject = $file ? "Processing of $file" : "";
2212 } ## end sub init_mail(;$)
2215 # finalize mail to be sent from msg(): check if something present, and
2220 debug("No mail for $main::mail_addr")
2221 if $main::mail_addr && !$main::mail_text;
2222 return unless $main::mail_addr && $main::mail_text;
2224 if ( !send_mail( $main::mail_addr, $main::mail_subject, $main::mail_text ) )
2227 # store this mail in memory so it isn't lost if executing sendmail
2230 @main::stored_mails,
2232 addr => $main::mail_addr,
2233 subject => $main::mail_subject,
2234 text => $main::mail_text
2237 } ## end if ( !send_mail( $main::mail_addr...
2240 # try to send out stored mails
2242 while ( $mailref = shift(@main::stored_mails) ) {
2244 !send_mail( $mailref->{'addr'}, $mailref->{'subject'},
2245 $mailref->{'text'} )
2248 unshift( @main::stored_mails, $mailref );
2250 } ## end if ( !send_mail( $mailref...
2251 } ## end while ( $mailref = shift(...
2252 } ## end sub finish_mail()
2257 sub send_mail($$$) {
2259 my $subject = shift;
2263 keys %main::packages ? join( ' ', keys %main::packages ) : "";
2267 unless ( defined($Email::Send::Sendmail::SENDMAIL) ) {
2268 $Email::Send::Sendmail::SENDMAIL = $conf::mail;
2271 if ($conf::overridemail) {
2272 $addr = $conf::overridemail;
2275 my $date = sprintf "%s",
2276 strftime( "%a, %d %b %Y %T %z", ( localtime(time) ) );
2277 my $message = <<__MESSAGE__;
2279 From: Debian FTP Masters <ftpmaster\@ftp-master.debian.org>
2286 if ( length $package ) {
2287 $message .= "X-Debian-Package: $package\n";
2290 $message .= "\n$text";
2291 $message .= "\nGreetings,\n\n\tYour Debian queue daemon (running on host $main::hostname)\n";
2293 my $mail = Email::Send->new;
2294 for (qw[Sendmail SMTP]) {
2295 $mail->mailer($_) and last if $mail->mailer_available($_);
2298 my $ret = $mail->send($message);
2299 if ( $ret && $ret !~ /Message sent|success/ ) {
2304 } ## end sub send_mail($$$)
2307 # try to find a mail address for a name in the keyrings
2309 sub try_to_get_mail_addr($$) {
2311 my $listref = shift;
2315 "$conf::gpg --no-options --batch --no-default-keyring "
2316 . "--always-trust --keyring "
2317 . join( " --keyring ", @conf::keyrings )
2321 if ( /^pub / && / $name / ) {
2323 push( @$listref, $1 );
2325 } ## end while (<F>)
2328 return ( @$listref >= 1 ) ? $listref->[0] : "";
2329 } ## end sub try_to_get_mail_addr($$)
2332 # return current time as string
2337 # omit weekday and year for brevity
2338 ( $t = localtime ) =~ /^\w+\s(.*)\s\d+$/;
2340 } ## end sub format_time()
2344 my $hours = int( $secs / ( 60 * 60 ) );
2346 $secs -= $hours * 60 * 60;
2347 return sprintf "%d:%02d:%02d", $hours, int( $secs / 60 ), $secs % 60;
2348 } ## end sub print_time($)
2351 # block some signals during queue processing
2353 # This is just to avoid data inconsistency or uploads being aborted in the
2354 # middle. Only "soft" signals are blocked, i.e. SIGINT and SIGTERM, try harder
2355 # ones if you really want to kill the daemon at once.
2357 sub block_signals() {
2358 POSIX::sigprocmask( SIG_BLOCK, $main::block_sigset );
2361 sub unblock_signals() {
2362 POSIX::sigprocmask( SIG_UNBLOCK, $main::block_sigset );
2366 # process SIGHUP: close log file and reopen it (for logfile cycling)
2373 open( LOG, ">>", $conf::logfile )
2374 or die "Cannot open my logfile $conf::logfile: $!\n";
2375 chmod( 0644, $conf::logfile )
2376 or msg( "log", "Cannot set modes of $conf::logfile: $!\n" );
2377 select( ( select(LOG), $| = 1 )[0] );
2379 open( STDOUT, ">&", \*LOG )
2381 "$main::progname: Can't redirect stdout to " . "$conf::logfile: $!\n" );
2382 open( STDERR, ">&", \*LOG )
2384 "$main::progname: Can't redirect stderr to " . "$conf::logfile: $!\n" );
2385 msg( "log", "Restart after SIGHUP\n" );
2386 } ## end sub close_log($)
2389 # process SIGCHLD: check if it was our statusd process
2394 # reap statusd, so that it's no zombie when we try to kill(0) it
2395 waitpid( $main::statusd_pid, WNOHANG );
2397 # Uncomment the following line if your Perl uses unreliable System V signal
2398 # (i.e. if handlers reset to default if the signal is delivered).
2399 # (Unfortunately, the re-setup can't be done in any case, since on some
2400 # systems this will cause the SIGCHLD to be delivered again if there are
2401 # still unreaped children :-(( )
2403 # $SIG{"CHLD"} = \&kid_died; # resetup handler for SysV
2404 } ## end sub kid_died($)
2406 sub restart_statusd() {
2408 # restart statusd if it died
2409 if ( !kill( 0, $main::statusd_pid ) ) {
2410 close(STATUSD); # close out pipe end
2411 $main::statusd_pid = fork_statusd();
2413 } ## end sub restart_statusd()
2416 # process a fatal signal: cleanup and exit
2418 sub fatal_signal($) {
2419 my $signame = shift;
2422 # avoid recursions of fatal_signal in case of BSD signals
2423 foreach $sig (qw( ILL ABRT BUS FPE SEGV PIPE )) {
2424 $SIG{$sig} = "DEFAULT";
2427 if ( $$ == $main::maind_pid ) {
2429 # only the main daemon should do this
2430 kill( $main::signo{"TERM"}, $main::statusd_pid )
2431 if defined $main::statusd_pid;
2432 unlink( $conf::statusfile, $conf::pidfile );
2433 } ## end if ( $$ == $main::maind_pid)
2434 msg( "log", "Caught SIG$signame -- exiting (pid $$)\n" );
2436 } ## end sub fatal_signal($)