Initial exit status tracking

diff --git a/ckandumper b/ckandumper
index 189459facdfe3d3f284d8ae1a3b29cd4ca0a26ca..736221998f8648af3c2aabc1e635da530a59db12 100755 (executable)
--- a/ckandumper
+++ b/ckandumper
@@ -84,9 +84,14 @@ class DownloadMultiple:
                     print(f'[stderr] {url} {stderr.decode()}')
 
             output = open(local_filename + '.returncode', 'w')
-            output.write(str(proc.returncode))
+            output.write(str(proc.returncode) + '\n')
             output.close()
 
+            if not str(proc.returncode) in self.stats['exitstatus']:
+                self.stats['exitstatus'][str(proc.returncode)] = []
+
+            self.stats['exitstatus'][str(proc.returncode)].append(url);
+
             if self.debug:
                 print(f'[{cmd!r} exited with {proc.returncode}]')
 
@@ -109,12 +114,18 @@ class DownloadMultiple:
         await asyncio.gather(*jobs)
 
     def get(self, filepairs):
-        self.bar = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False
-        loop     = asyncio.get_event_loop()
+        self.stats = { 'exitstatus': {} }
+        self.bar   = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False
+        loop       = asyncio.get_event_loop()
 
         loop.set_debug(self.debug)
         loop.run_until_complete(self.gather(filepairs))
 
+        if self.debug:
+            print(self.stats)
+
+        return self.stats
+
 class CkanDumper:
     """Dumps CKAN data: metadata plus entire datasets"""
 
@@ -173,7 +184,7 @@ class CkanDumper:
         # Groups
         #
         print(f'Downloading {self.url}{self.group_list}...')
-        self.download.get([[self.url + self.group_list, group_list]])
+        status = self.download.get([[self.url + self.group_list, group_list]])
 
         groups          = self.load_json(group_list)
         group_downloads = []
@@ -185,13 +196,13 @@ class CkanDumper:
 
             group_downloads.append([self.url + self.group_show + urlencode({ 'id': group }, False, '', 'utf-8'), group_file])
 
-        self.download.get(group_downloads)
+        status = self.download.get(group_downloads)
 
         #
         # Tags
         #
         print(f'Downloading {self.url}{self.tag_list}...')
-        self.download.get([[self.url + self.tag_list, tag_list]])
+        status = self.download.get([[self.url + self.tag_list, tag_list]])
 
         tags           = self.load_json(tag_list)
         tags_downloads = []
@@ -203,13 +214,13 @@ class CkanDumper:
 
             tags_downloads.append([self.url + self.tag_show + urlencode({ 'id': tag }, False, '', 'utf-8'), tag_file])
 
-        self.download.get(tags_downloads)
+        status = self.download.get(tags_downloads)
 
         #
         # Packages
         #
         print(f'Downloading {self.url}{self.package_list}...')
-        self.download.get([[self.url + self.package_list, package_list]])
+        status = self.download.get([[self.url + self.package_list, package_list]])
 
         packages           = self.load_json(package_list)
         packages_downloads = []
@@ -222,7 +233,7 @@ class CkanDumper:
 
             packages_downloads.append([self.url + self.package_show + urlencode({ 'id': package }, False, '', 'utf-8'), package_file])
 
-        self.download.get(packages_downloads)
+        status = self.download.get(packages_downloads)
 
         #
         # Package contents
@@ -251,7 +262,7 @@ class CkanDumper:
 
                 package_downloads.append([resource['url'], resource_file])
 
-            self.download.get(package_downloads)
+            status = self.download.get(package_downloads)
 
 if __name__ == "__main__":
     # Parse CLI