Simplify more resource processor

whiletrue-industries · Jun 12, 2024 · f2f63f1 · f2f63f1
1 parent bd1c36d
commit f2f63f1
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/odds/backend/processor/resource_processor.py b/odds/backend/processor/resource_processor.py
@@ -159,12 +159,15 @@ async def process(self, resource: Resource, dataset: Dataset, catalog: DataCatal
                             if len(true_values) == 0:
                                 continue
                             resource.fields.append(field)
-                            field_names.append(col_name)
                             try:
                                 field.sample_values = [str(x) for x, _ in Counter(true_values).most_common(10)]
+                                if len(field.sample_values) == 1:
+                                    # if all values are the same, no need for this field in the db
+                                    continue
                             except:
                                 pass
-                            if len(values) > 0:
+                            field_names.append(col_name)
+                            if len(values) > 0 and len(true_values) != len(values):
                                 field.missing_values_percent = int(100 * (len(values) - len(true_values)) / len(values))
                             if field.data_type in ('number', 'integer', 'date', 'time', 'datetime'):
                                 true_values = set(true_values)