Adding httracker_iterate

diff --git a/httracker b/httracker
index 5b5de29618cdfd16f0ffeb06f1fa5a4a4496cec8..385035c3bc9b596d0e15f4ed2eeded8ae55779e4 100755 (executable)
--- a/httracker
+++ b/httracker
@@ -22,9 +22,7 @@ if [ "$?" != "0" ]; then
 fi
 
 # Iterate over all URLs
-for link in `cat $URLS | xargs`; do
-  httracker_get "$link"
-done
+httracker_iterate
 
 # Cleanup
 rm -rf $TMP

diff --git a/lib/httracker/functions b/lib/httracker/functions
index ea9a7fa1fc59463271e5fb8838048eb8b2c47200..c7dff7f1c6c13f9772198655733339994e1cdcc0 100644 (file)
--- a/lib/httracker/functions
+++ b/lib/httracker/functions
@@ -41,27 +41,25 @@ function httracker_get {
   fi
 }
 
-function httracker_get_single {
-# grabs URLs from the network
+function httracker_get_incremental {
+  # Creates target dir
+  year=`date +%Y`
+  month=`date +%m`
+  %day=`date +%d`
+  target=${MIRRORDIR}/${year}/${month}
+  sudo -u links mkdir -p ${target}
+
+  # Grabs URLs from the network
   httrack --verbose           \
           --user links        \
           --depth=1           \
           --purge-old=0       \
           --index             \
           --cookies=1         \
-          --list ${tmpfile}   \
+          --list ${URLS}   \
           --path ${target} \
 }
 
-function httracker_target_single {
-  # creates target dir
-  year=`date +%Y`
-  month=`date +%m`
-  %day=`date +%d`
-  target=${MIRRORDIR}/${year}/${month}
-  sudo -u links mkdir -p ${target}
-}
-
 function httracker_scuttle_config() {
   grep ${1} ${CONFIGFILE} | sed -e s/\[^\'\]\*\'// -e s/\'\.\*\$//
 }
@@ -74,3 +72,9 @@ httracker_sqlquery() {
         --host=${dbhost}          \
         --execute="${1}"
 }
+
+function httracker_iterate {
+  for link in `cat $URLS | xargs`; do
+    httracker_get "$link"
+  done
+}

diff --git a/scuttler b/scuttler
index d846d87b66753fe7618a49448993cf4e41c9246f..0b2b6dae356939df71779f78638530a17a1b4d05 100755 (executable)
--- a/scuttler
+++ b/scuttler
@@ -25,9 +25,11 @@ chown links.links ${URLS}
 chmod 600 ${URLS}
 httracker_sqlquery "select bAddress from sc_bookmarks;" > ${URLS}
 
-# Prepare and download
-httracker_target_single
-httracker_get_single
+# Download in incremental mode
+httracker_get_incremental
+
+# Download in mirror mode
+httracker_iterate
 
 # Cleanup
 rm -rf $TMP