More reports

whiletrue-industries · Jun 11, 2024 · 7d548dc · 7d548dc
1 parent cf9fa3e
commit 7d548dc
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/odds/backend/processor/resource_processor.py b/odds/backend/processor/resource_processor.py
@@ -79,10 +79,14 @@ async def process(self, resource: Resource, dataset: Dataset, catalog: DataCatal
                                     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0'
                                 }
                                 headers.update(catalog.http_headers)
+                                report = 0
                                 async with client.stream('GET', resource.url, headers=headers, timeout=60, follow_redirects=True) as response:
                                     async for chunk in response.aiter_bytes():  
                                         f.write(chunk)
                                         total_size += len(chunk)
+                                        while total_size - report > 1000000:
+                                            report += 1000000
+                                            rts.set(ctx, f'DOWNLOADED {report} BYTES from {resource.url} to {filename}')
 
                         rts.set(ctx, f'DOWNLOADED {total_size} BYTES from {resource.url} to {filename}')
                         dp, _ = DF.Flow(