#!/bin/bash -e

# A complex script that creates a repository of deltas, that can be
# used by debdelta-upgrade for upgrading packages.
# See also --help below.

# WARNING : after the release of Squeeze, this script is deprecated:
#  indeed the new archive signatures need debmirror >= 2.6,
#  and I did not prepare a 'trash' patch for it, but instead wrote
#  a new script to use the new 'debmirror --marshal' option

# Copyright (C) 2006-11 Andrea Mennucci.
# License: GNU Library General Public License, version 2 or later


########### customize under here  vvvvvvvvvvvvv

#where is the full mirror of debian
debmir=/mirror/debian

#where to mirror from
host=ftp.debian.org

#where to store deltas
deltamir=/mirror/debian-deltas
#delete old debs from trash if the space here drops below kB :
minfreespace=2000000
#delete deltas and old_debs that are older than days (regardless of space)
old_delete_days=50

#where to keep snapshots of the older indexes of the archives (for the --snapshot option)
dists_snapshots=/mirror/debian-snapshots

#where the gnupg stuff specific to debdelta resides
# and in particular the public and private keyrings
GNUPGHOME=/var/lib/debdelta/gnupg

#where the gnupg agent variable is stored
#(unset this if you do not want to use the agent)
# note that this needs gnupg2 >= 2.0.11 (that is not lenny)
GNUPGAGENTINFO="${GNUPGHOME}/debdeltas-gpg-agent-info" 

#the secret key to sign the deltas
GNUPGSEC=0xTHESECKEY

#note: if you export this , then it will affect also debmirror:
# make sure the deb repository key is copied there
#export GNUPGHOME

#where is the debdeltas program
debdeltas=/usr/bin/debdeltas
#options to your taste
debdelta_opt="-v -d --test --delta-algo xdelta3 --gpg-home $GNUPGHOME --sign $GNUPGSEC "

##if you apply the 'trash patch' in contrib to debmirror, then you may use this:
debmirror_opt="--trash $deltamir/old_debs --method=http  --diff=none"
trash=$deltamir/old_debs
debmirror=/usr/local/bin/debmirror_2.4_mine

##otherwise this should work as well
#debmirror_opt='--method=http'
#trash=''
#debmirror=/usr/bin/debmirror

#note: files in $trash will be deleted by this script! use a dedicated dir!

#this should be in the same partition as the debian mirror $debmir
# (must be, if the trash is not used)
TMPDIR=/mirror/tmp
export TMPDIR


ARCHc='i386,amd64'
ARCHs='i386 amd64'

DISTc='lenny,squeeze,sid,experimental'
DISTs='lenny squeeze sid experimental'

########### customize over here ^^^^^^^^^^^^^^


#since the output is automatically parsed, do it in English 
unset LANG
unset LC_ALL
unset LC_NUMERIC

startgpgagent () {
 gpg-agent --homedir "${GNUPGHOME}" --daemon --write-env-file "$GNUPGAGENTINFO" ; 
}

##enable this after catastrofic failures, to try to create all leftover deltas
## parsing all olddists once again
RECOVER=''

DEBUG=''
VERBOSE=''
RM='rm'
DO_MIRROR=true
DO_CLEAN_DELTAS=''
DO_SNAPSHOT=''
while [ "$1" ] ; do
 case "$1" in
  -h|--help)
  cat <<EOF
Usage: debmirror-deltas [options]

This script (that contains many variables that must be customized!)
keeps a full mirror of the Debian repository (using 'debmirror'),
and it uses it to build a repository of deltas (using 'debdeltas').

Note that this script assumes that the current Debian
distributions are $DISTs .
It will generate deltas for the upgrades in those,
on arches $ARCHs .

Options:

-v   verbose   (passed to debmirror and debdeltas)
-d   debug     (passed to debmirror and debdeltas),
               also, do not 'rm' stuff, just tell
--no-mirror    do not download debs to the local mirror
--recover      if it crashed, resume the unfinished jobs
--clean-deltas clean useless deltas (see debdeltas man page)
--snapshot     capture a snapshot of the archive (use daily)
--start-gpg-agent               start gpg agent, feed
               the passwd for the secret gpg key in it
EOF
  exit ;;
  -v)  VERBOSE="$VERBOSE -v"  ;;
  -d)  DEBUG=--debug ;  RM='echo -- would rm ' ;;
  #used to finish interrupted runs
  --no-mirror) DO_MIRROR='' ;;
  --recover) RECOVER=true ;;
  --clean-deltas) DO_CLEAN_DELTAS=true ;;
  --snapshot) DO_SNAPSHOT=true ;;
  --start-gpg-agent)
     echo ------  start agent
     startgpgagent
     . "$GNUPGAGENTINFO" 
     export GPG_AGENT_INFO
     #force loading of the private key (I know no better way)
     echo ------ sign dummy file, to load the key
     t=`tempfile`
     echo pippo > $t
     gpg2 --quiet --homedir "${GNUPGHOME}" -o /dev/null --sign $t
     rm $t
     exit 0
    ;;
  *) echo "Unknown option $1 , try --help" ;  exit 1 ;;
 esac
 shift
done

#################### set gpg stuff, test it


gpgagentcmd="$0 --start-gpg-agent"

# set gpg-agent variables, test it
if test "$GNUPGAGENTINFO"  ; then 
 if test ! -r "$GNUPGAGENTINFO"  ; then 
  echo ERROR no agent info, please start the agent with
  echo $gpgagentcmd
  exit 1
 else
  . "$GNUPGAGENTINFO" 
  export GPG_AGENT_INFO
  if test ! "${GPG_AGENT_INFO}" -o  ! -e "${GPG_AGENT_INFO/:*/}" -o ! -O "${GPG_AGENT_INFO/:*/}" ; then
     echo ERROR agent info is not OK, please run the command
     echo $gpgagentcmd
     exit 1
  elif ! echo | gpg-connect-agent --homedir ${GNUPGHOME} ; then
     echo ERROR agent is not responding, please run the command
     echo $gpgagentcmd
     exit 1
  fi
 fi
fi


#test that we can sign, possibly loading the password in the agent
if test  "$GNUPGSEC" ; then
 t=`tempfile`
 echo pippo > $t
 if  ! gpg2 --quiet --batch --homedir "${GNUPGHOME}"  -o /dev/null --default-key $GNUPGSEC --sign $t  ;
 then
  echo signature test FAILED
  rm $t
  exit 1
 fi
 rm $t
fi

###################### some useful variables

#the lock used by debmirror
debmirlock=$debmir/Archive-Update-in-Progress-`hostname  -f`

b=`basename $0`

lockfile -r 10  /tmp/$b.lock || exit 1
trap "rm $VERBOSE -f /tmp/$b.lock" 0

olddists=""
log=/nonexistant

## profile debdeltas
#debdeltas="python -m cProfile -o /tmp/debdelta-profile-$(dirname $p | tr / _ ) $debdeltas"

today=`date +'%F'`
yyyymm=`date +'%Y-%m'`

################## routines

freetrash () {
 test "$trash" || return
 age=${old_delete_days}
 blocksize=`stat -f  -c "%s" "$trash"`
 newsize=0 #trick to run at least once
 while test $age -ge 1 -a $newsize -le $minfreespace ; do
  find "$trash"  -name '*.deb' -type f -mtime +$age  |  xargs -r $RM $VERBOSE || true
  freeblocks=`stat -f -c "%a" "$trash"`
  newsize=`expr $freeblocks \* $blocksize / 1024`
  age=`expr $age - 5`
 done
}

cleanuppool () {
 find $deltamir/pool \
   \( -name '*debdelta-fails' -or -name '*debdelta-too-big' \
      -or -name  '*debdelta' \) -mtime +${old_delete_days} -type f |\
       xargs -r  $RM   $VERBOSE || true
 
 find $deltamir/pool/ -type d -empty | xargs -r rmdir $VERBOSE || true

 if test "$trash" ; then 
  find $trash -type f -not -name '*.deb' | xargs -r  $RM  $VERBOSE || true
 fi
}



run_debmirror () {
 if test -e $debmirlock ; then 
  echo Archive $debmir is locked 
  exit 1
 fi

 olddists=${TMPDIR:-/tmp}/olddists-`date +'%F_%H-%M-%S'`
 mkdir $olddists

 tm=$olddists/debmirror-stdout
 tme=$olddists/debmirror-stderr

 cp -a $debmir/dists $olddists

 ## if using trash, this is not needed : debdeltas will
 ##go fishing in the trash dir to find all deleted deltas
 if ! test "trash"  ; then
  cp -al $debmir/pool $olddists
 fi

 $debmirror  $debmir --nosource \
  $debmirror_opt  --arch=$ARCHc \
  --section=main,contrib,non-free \
  -v $DEBUG -h $host -d $DISTc \
     > $tm 2> $tme ; dme=$? 

 if test "$dme" != 0  ; then
   echo debmirror failed , exit code $dme , stdout  $tm 
   awk '{ print "> " $0 }' $tm 
   echo debmirror failed stderr $tme 
   awk '{ print "> " $0 }' $tme
   $RM -f $debmirlock || true
   #sometimes the temporary directory of debmirror is messed up
   #rm -r $debmir/.temp
   cleanuppool
   freetrash
   exit 1
 else
   $RM $VERBOSE $tm $tme
 fi
}



findmelog () {
 date=`date +'%F-%H'`
 log=$deltamir/log/$yyyymm/${date}.log
 err=$deltamir/log/$yyyymm/${date}.err
 if test -r $log -o -r $log.gz -o -r $err -o -r $err.gz ; then 
  for i in 0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z ; do 
   if test -r $log -o -r $log.gz  -o -r $err -o -r $err.gz ; then 
     log=$deltamir/log/$yyyymm/${date}.$i.log
     err=$deltamir/log/$yyyymm/${date}.$i.err
   fi
  done
 fi
}

created_deltas=""

run_debdeltas  () {
 if ! test "$olddists" -a -r "$olddists/dists" ; then
  echo "run_debdelta : bad $olddists"
  return
 fi
 findmelog
 (
  exec >> $log
  exec 2>> $err
  echo -n --------------oldnew pass----   ; date   ;
  echo "------ tmpdir was in $olddists"
  echo -n ---- options to debdelta ;  echo -- $debdelta_opt

  for dist in $DISTs ; do
   echo -------------- $dist ----
   #lenny contains debdelta 0.27, that does not understand lzma
   if test "$dist" = lenny ; then x="--disable-feature lzma" ; else x="" ; fi
   for p in $( cd $debmir ; find dists/$dist/ -name Packages.gz) ; do
    if test -r ${dists_snapshots}/before.4/$p ; then o="--old  ${dists_snapshots}/before.4/$p" ; else o='' ; fi
    echo ------ debdelta $x $o --old $olddists/$p $p
    $debdeltas $VERBOSE $debdelta_opt  \
       --alt $trash $x $o \
       --old $olddists/$p \
       --dir $deltamir//  $debmir/$p
   done
  done
 )

 #uncomment for added verbosity
 #egrep -3 -i 'error|warning' $log || true

 if  test -s $err || grep -iq 'error' $log ; then
  #some error occurred
  test ! -r "$log".gz && { gzip -9 $log ; log=${log}.gz ; }
  echo "-----ERRORS--- full log is in $log  or ...err.gz , data in $olddists "
 else
  rm $err # it is empty
  #no error occurred, clean up
  if test "$olddists" ; then
   if test -r "$olddists/dists" ; then
    $RM -r  "$olddists/dists"
   fi
   if test -r "$olddists/pool" ; then
    $RM -r  "$olddists/pool"
   fi
   #this is a bit exotic , but unfortunately 'test' does not test empty dirs
   if find "$olddists" -maxdepth 0 -type d -empty | grep -q "$olddists" ; then
    rmdir "$olddists" ; else echo Not empty "$olddists" ; 
   fi
  fi
  if grep -v '^---' $log | grep -v '^ Total running time' |  grep  -q '.' ; then
   test ! -r "$log".gz && gzip -9 $log
   created_deltas=1
  else
   $RM $log
  fi
 fi
}


run_debdeltas_clean () {
   echo -------------- cleanup delta pool
   $debdeltas --clean-deltas -n 0 \
     --dir $deltamir// \
       $(  for arch in $ARCHs ; do
            for sec in main contrib non-free ; do
              for dist in $DISTs ; do
                echo $debmir/dists/$dist/$sec/binary-$arch/Packages.gz 
           done ; done ; done )
}


########################################### code

#always free space from the trash
freetrash

if test `stat -f -c "%a" "$deltamir"` -le 32 ; then
 echo "------- emergency pool cleanup , very low disk space in delta mirror"
 run_debdeltas_clean
 cleanuppool
elif test "${DO_CLEAN_DELTAS}" ; then
 run_debdeltas_clean
 cleanuppool
fi

test -d $deltamir/log/$yyyymm ||  mkdir $deltamir/log/$yyyymm

if test "${DO_SNAPSHOT}" ; then
  if test -e $debmirlock ; then 
   echo Archive $debmir is locked, cannot --snapshot 
  elif test -w ${dists_snapshots} -a -d ${dists_snapshots} ; then
   test -e ${dists_snapshots}/before.4 && rm -r ${dists_snapshots}/before.4
   test -e ${dists_snapshots}/before.3 && mv ${dists_snapshots}/before.3 ${dists_snapshots}/before.4
   test -e ${dists_snapshots}/before.2 && mv ${dists_snapshots}/before.2 ${dists_snapshots}/before.3
   test -e ${dists_snapshots}/before.1 && mv ${dists_snapshots}/before.1 ${dists_snapshots}/before.2
   mkdir ${dists_snapshots}/before.1
   cp -a $debmir/dists ${dists_snapshots}/before.1/.
  else
   echo Please set dists_snapshots to a directory where I can write, not ${dists_snapshots} 1>&2
  fi
fi

if [ "${DO_MIRROR}" ] ; then
 run_debmirror
else
 echo ---------- skipped mirror step
fi

if [ "$olddists"  -a -r "$olddists" ] ; then 
 #the mirror was updated, so we check the difference 
 run_debdeltas
else
 test "$DEBUG" && echo olddists is not defined or is not readable : "$olddists"
fi

##enable this to try to create all leftover deltas
if test "$RECOVER" ; then
 for olddists  in ${TMPDIR:-/tmp}/olddists* ; do
  if [ "$olddists"  -a -r "$olddists" ] ; then
   echo ----------------------------------- creating deltas wrt $olddists
   run_debdeltas
  fi
 done
fi
