]> gitweb.fluxo.info Git - ckandumper.git/commitdiff
Put all package contents in a single async loop
authorSilvio Rhatto <rhatto@riseup.net>
Thu, 16 May 2019 16:49:59 +0000 (13:49 -0300)
committerSilvio Rhatto <rhatto@riseup.net>
Thu, 16 May 2019 16:49:59 +0000 (13:49 -0300)
ckandumper

index bfb99d7cf0e12f04c1390a50bf0057109c3d16fe..46c9a51a0501bf8ed3989584c734e4157a91e1a6 100755 (executable)
@@ -263,6 +263,8 @@ class CkanDumper:
         stats = self.download.get(packages_downloads)
         self.process_stats(stats)
 
+        print('Downloading contents of all packages...')
+
         #
         # Package contents
         #
@@ -272,8 +274,6 @@ class CkanDumper:
             package_file      = package_folder + os.sep + 'package.json'
             contents          = self.load_json(package_file)
 
-            print(f'Downloading contents of package {package}...')
-
             for resource in contents['result']['resources']:
                 #if resource['name'] != None:
                 #  name = resource['name']
@@ -290,8 +290,8 @@ class CkanDumper:
 
                 package_downloads.append([resource['url'], resource_file])
 
-            stats = self.download.get(package_downloads)
-            self.process_stats(stats)
+        stats = self.download.get(package_downloads)
+        self.process_stats(stats)
 
 if __name__ == "__main__":
     # Parse CLI