Adds --randomize

diff --git a/ckandumper b/ckandumper
index 7915c3b89961cab5b5893cec7a114a50ba946feb..f63321509175ba977de138da66c49bc8df725eb3 100755 (executable)
--- a/ckandumper
+++ b/ckandumper
@@ -162,6 +162,9 @@ class CkanDumper:
         if args.progress != None:
             self.progress = args.progress
 
+        if args.randomize != None:
+            self.randomize = args.randomize
+
         if args.limit_rate != None:
             self.limit_rate = '--limit-rate=' + args.limit_rate
         else:
@@ -296,22 +299,28 @@ class CkanDumper:
 
                 package_downloads.append([resource['url'], resource_file])
 
+        if self.randomize == True:
+            random.shuffle(package_downloads)
+
         stats = self.download.get(package_downloads)
         self.process_stats(stats)
 
 if __name__ == "__main__":
     # Parse CLI
     parser = argparse.ArgumentParser(description='Dump CKAN metadata and datasets.')
-    parser.add_argument('url',                nargs='+',                             help='CKAN instance URL')
-    parser.add_argument('dest',               nargs='+',                             help='Destination folder')
-    parser.add_argument("--limit-rate",                                              help="Limit the download speed to amount bytes per second, per download")
-    parser.add_argument("--limit-concurrent",                                        help="Limit the total concurrent downloads")
-    parser.add_argument("--wget",                                                    help="Path of custom wget implementation")
-    parser.add_argument('--debug',            dest='debug',    action='store_true',  help="Enable debug")
-    parser.add_argument('--no-debug',         dest='debug',    action='store_false', help="Disable debug")
-    parser.add_argument('--progress',         dest='progress', action='store_true',  help="Enable progress")
-    parser.add_argument('--no-progress',      dest='progress', action='store_false', help="Disable progress")
+    parser.add_argument('url',                nargs='+',                                help='CKAN instance URL')
+    parser.add_argument('dest',               nargs='+',                                help='Destination folder')
+    parser.add_argument("--limit-rate",                                                 help="Limit the download speed to amount bytes per second, per download")
+    parser.add_argument("--limit-concurrent",                                           help="Limit the total concurrent downloads")
+    parser.add_argument("--wget",                                                       help="Path of custom wget implementation")
+    parser.add_argument('--debug',            dest='debug',     action='store_true',    help="Enable debug")
+    parser.add_argument('--no-debug',         dest='debug',     action='store_false',   help="Disable debug")
+    parser.add_argument('--progress',         dest='progress',  action='store_true',    help="Enable progress")
+    parser.add_argument('--no-progress',      dest='progress',  action='store_false',   help="Disable progress")
+    parser.add_argument('--randomize',        dest='randomize', action='store_true',    help="Randomize the list of downloads to avoid consuming resources of the same remote endpoint")
+    parser.add_argument('--no-randomize',     dest='randomize', action='store_false',   help="Do not randomize the list of downloads")
     parser.set_defaults(debug=False)
+    parser.set_defaults(randomize=False)
     parser.set_defaults(progress=True)
     args = parser.parse_args()