#!/bin/bash
# 
# update SquidGuard blacklists
#
# run this weekly from a cron job
#
# author: T.F.Harris
# email: tfhlonbi.co.uk
# date: June 2013
#

TMPDIR=/tmp
SQUIDG_DB=/var/db/squidGuard/
SQUIDG_CONF=/usr/local/etc/squid/squidGuard.conf
SQUIDG_USER=squid
SQUIDG_GROUP=${SQUID_USER}
SQUIDG_CMD=/usr/local/bin/squidGuard
SQUIDG_ARGS="-b -d -P -C all"

# configure wget. If you go out through a proxy, set the
# username and pasword
proxy_user=blacklist 
proxy_pass=password
WGET=/usr/local/bin/wget
WGET_ARGS="--proxy-user=$proxy_user --proxy-password=$proxy_pass -nv"

#
# commands to rebuild the squidGuard Database
# and reload the squid servive
#
SQUID_RELOAD_CMD="service squid reload"
SQUIDG_RELOAD_CMD="${SQUIDG_CMD} ${SQUIDG_ARGS}"

#
# temporary directoery to unpack the lists
# this will be deleted at the end
bldir=${TMPDIR}/bl${RANDOM}

blroot=${bldir}

#
# the files used in the squidguard db
#
filelist="domains urls expressions"
#
# list of blacklist sources 
blacklists=$(cat<<"SRC" http://squidguard.mesd.k12.or.us/blacklists.tgz http://www.shallalist.de/Downloads/shallalist.tar.gz SRC ) #http://urlblacklist.com/cgi-bin/commercialdownload.pl?type=download&file=bigblacklist get_blacklist() {     local bl     local dest      bl=$1     dest=$2     pushd $dest     echo getting $bl and storing in $dest     eval '$WGET $WGET_ARGS $bl'     popd } merge_files () {     #     # commbine the existing squidGuard file and the      # new blacklist file.     # sort & uniq the union and then      # put back in the squidGuard db location     #     # ags:     # full path spec of the squidGuard file      # full path of the new black list file     #     local squidfile # existing squid file     local newfile # from the new blacklist     local tmpfile     tmpfile=${TMPDIR}/${RANDOM}.txt     squidgfile=$1     newfile=$2     touch  $squidgfile     touch  $newfile      echo combining ${squidgfile} ${newfile}      cat ${squidgfile} ${newfile} | sort -fb | uniq -i > $tmpfile
    if [ -e $squidgfile ] 
    then
        rm -f ${squidgfile}
    fi

    mv ${tmpfile} ${squidgfile}
}

unpack_blacklist ()
{
    #
    # unpack the archive
    # alters blroot for the root directory of the blacklists
    # becuase it may be several layers down
    #
    local blacklist
    local tmpdir
    blacklist=$1
    tmpdir=$2
    echo unpacking archive ${blacklist} to ${tmpdir}
    pushd ${tmpdir}
    tar xzf $blacklist
    popd
    blroot=$(dirname $(dirname $(find ${tmpdir} -type f -name "domains" | head -1)))

    echo black list root directory is $blroot
}

cleanup () 
{
    #
    # all post processing cleanups
    # remove the temporary directory used to
    # hold the un tarred list
    local tmpdir
    tmpdir=$1
    rm -rf $tmpdir
}

pre_check () 
{
    #
    # any checks for the enviroment etc to go here
    # if returns 0 - good to go
    #
    local retval
    retval=0
    if [ ! -d $SQUIDG_DB ] 
    then
        mkdir -p $SQUIDG_DB
    fi
    if [ ! -d $SQUIDG_DB ] 
    then
        retval=1
    fi
    touch ${TMPDIR}/all.txt
    all_okay=$retval
    return 
}

alter_conf () 
{
    local alldirs
    local fulllist
    local conftemp
    alldirs=${TMPDIR}/all.txt
    fulllist=${TMPDIR}/full.txt
    conftemp=${TMPDIR}/sqg_conf.txt

    #
    # add all the database files to the configuration
    # this is a catch all to ensure everything is processed
    # ignore all db files - 
    #

    # create the new section for the squidGaudr.conf file
    #
    echo '### AUTO GENERATED CONFIG ###'  > ${alldirs}
    echo '### generated on ' $(date) >> ${alldirs}
    echo '###'  >> ${alldirs}
    echo '###' >> ${alldirs}
    echo '###---' >> ${alldirs}

    find ${SQUIDG_DB} -depth 2 -type f | grep -v '.db$' | sort >> ${alldirs}

    # turn the list
    # of files into the format required by squidGuard
    # and place in a automatic dest block
    #
    #/^vvvvvvdest/   s/vvvvvvdest/dest auto_bl_complete {/i 
    sed  -e '
    {
        /###---/ a\
            dest auto_bl_complete {

        /^[^#]/ {
        s/^\.//
        s#\(.*\)/\(.*\)s#    \2list \1/\2s# 
    }
    $ a\ 
    }

}
' < ${alldirs} > ${fulllist}

#
# remove the old lines from the conf file.
# delete every thing from the auto conf heading to the end of the file
#
sed -e '/### AUTO GENERATED CONFIG ###/,$ d' < ${SQUIDG_CONF} > ${conftemp}

# re-generate the config file
cat ${conftemp}  ${fulllist} > ${SQUIDG_CONF} 
rm ${alldirs} ${fulllist} ${conftemp}

}

reload_daemons ()
{
    chown -R ${SQUIDiG_USER}:${SQUIDG_GROUP} ${SQUIDG_DB}
    eval '$SQUIDG_RELOAD_CMD'
    eval '$SQUID_RELOAD_CMD'
}

sub=0

allokay=0
pre_check 
if [  $all_okay -ne 0 ] 
then
    echo error
else

    mkdir -p $bldir
    for blget in ${blacklists}
    do
        blroot=""
        echo ======================================
        echo Processing blacklist: $blget
        echo ======================================
        let sub=$sub+1
        blsave=${bldir}/${sub}
        mkdir -p ${blsave}
        pushd ${blsave}
        echo get_blacklist ${blget} ${blsave} 
        get_blacklist ${blget} ${blsave}

        blfile=$(find ./ -type f -depth 1 | head -1)

        echo unpack_blacklist ${blsave}/${blfile} ${blsave}
        unpack_blacklist ${blsave}/${blfile} ${blsave}
        popd

        echo blroot is $blroot
        pushd ${blroot}
        #
        # check that there is a squid database for each of the blacklists
        #
        for f in * 
        do  
            if [ -d $f ] 
            then
                destdir=${SQUIDG_DB}/${f}
                echo $f is a directory destination is ${destdir}

                if  [ ! -d ${destdir} ] 
                then
                    echo ${destdir} does not exist: creating directory
                    mkdir -p $destdir
                    for file in ${filelist}
                    do 
                        squidgdb=${destdir}/${file}
                        echo creating file ${squidgdb}
                        touch ${squidgdb}
                    done
                fi

                if  [ -d ${destdir} ]
                then
                    for file in ${filelist}
                    do 
                        squid_db=${destdir}/${file}
                        blacklist=./${f}/${file}
                        touch ${squid_db}
                        echo merging ${blacklist} and ${squid_db}
                        merge_files ${squid_db} ${blacklist}
                    done
                fi

            fi 

        done
        popd

        echo completed processing blacklist: $blget
        echo ======================================
        mv $blroot ${blroot}_processed_${sub}
    done

    alter_conf 
    reload_daemons 
    cleanup ${bldir}

fi