diff --git a/app/backend/gwells/db_comments/db_actions.py b/app/backend/gwells/db_comments/db_actions.py index 929f33dc78..5f743789fa 100644 --- a/app/backend/gwells/db_comments/db_actions.py +++ b/app/backend/gwells/db_comments/db_actions.py @@ -57,7 +57,7 @@ def create_db_comments_from_models(models): for model_class in models: # Doing the check for abstract is a bit weird, we have to create an instance of the class, we # can't check it on the class definition. - if model_class()._meta.abstract: + if model_class._meta.abstract: # If it's an abstract class, don't proceed. continue diff --git a/app/backend/gwells/migrations/0001_initial.py b/app/backend/gwells/migrations/0001_initial.py index d4f08f13f5..0732a20226 100644 --- a/app/backend/gwells/migrations/0001_initial.py +++ b/app/backend/gwells/migrations/0001_initial.py @@ -63,11 +63,11 @@ class Migration(migrations.Migration): name='Border', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('se_a_c_flg', models.CharField(max_length=254)), + ('se_a_c_flg', models.CharField(max_length=254, null=True)), ('obejctid', models.FloatField()), - ('shape', models.FloatField()), + ('shape', models.FloatField(null=True)), ('length_m', models.FloatField()), - ('oic_number', models.CharField(max_length=7)), + ('oic_number', models.CharField(max_length=7, null=True)), ('area_sqm', models.FloatField()), ('upt_date', models.CharField(max_length=20)), ('upt_type', models.CharField(max_length=50)), @@ -78,8 +78,8 @@ class Migration(migrations.Migration): ('aa_name', models.CharField(max_length=100)), ('abrvn', models.CharField(max_length=40)), ('bdy_type', models.CharField(max_length=20)), - ('oic_year', models.CharField(max_length=4)), - ('afctd_area', models.CharField(max_length=120)), + ('oic_year', models.CharField(max_length=4, null=True)), + ('afctd_area', models.CharField(max_length=120, null=True)), ('geom', django.contrib.gis.db.models.fields.MultiPolygonField(srid=4269)), ], ), diff --git a/app/backend/gwells/migrations/0009_auto_20240605_1836.py b/app/backend/gwells/migrations/0009_auto_20240605_1836.py new file mode 100644 index 0000000000..5a07c82393 --- /dev/null +++ b/app/backend/gwells/migrations/0009_auto_20240605_1836.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2 on 2024-06-05 18:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gwells', '0008_profile_silver_keycloak_id'), + ] + + operations = [ + migrations.AlterField( + model_name='border', + name='afctd_area', + field=models.CharField(max_length=120, null=True), + ), + migrations.AlterField( + model_name='border', + name='oic_number', + field=models.CharField(max_length=7, null=True), + ), + migrations.AlterField( + model_name='border', + name='oic_year', + field=models.CharField(max_length=4, null=True), + ), + migrations.AlterField( + model_name='border', + name='se_a_c_flg', + field=models.CharField(max_length=254, null=True), + ), + migrations.AlterField( + model_name='border', + name='shape', + field=models.FloatField(null=True), + ), + ] diff --git a/app/backend/gwells/models/__init__.py b/app/backend/gwells/models/__init__.py index 6f7fc12b9b..6987217f88 100644 --- a/app/backend/gwells/models/__init__.py +++ b/app/backend/gwells/models/__init__.py @@ -53,11 +53,11 @@ class Meta: class Border(models.Model): - se_a_c_flg = models.CharField(max_length=254) + se_a_c_flg = models.CharField(max_length=254, null=True) obejctid = models.FloatField() - shape = models.FloatField() + shape = models.FloatField(null=True) length_m = models.FloatField() - oic_number = models.CharField(max_length=7) + oic_number = models.CharField(max_length=7, null=True) area_sqm = models.FloatField() upt_date = models.CharField(max_length=20) upt_type = models.CharField(max_length=50) @@ -68,8 +68,8 @@ class Border(models.Model): aa_name = models.CharField(max_length=100) abrvn = models.CharField(max_length=40) bdy_type = models.CharField(max_length=20) - oic_year = models.CharField(max_length=4) - afctd_area = models.CharField(max_length=120) + oic_year = models.CharField(max_length=4, null=True) + afctd_area = models.CharField(max_length=120, null=True) geom = models.MultiPolygonField(srid=4269) diff --git a/app/backend/requirements.txt b/app/backend/requirements.txt index ed99f4db0f..8cb7be2aa8 100644 --- a/app/backend/requirements.txt +++ b/app/backend/requirements.txt @@ -1,33 +1,36 @@ -django>=2.2,<2.3 -django-debug-toolbar>=1.7 +django==3.2.4 +pygdal==3.6.2.11 +gdal==3.0.0 +pytz==2024.1 +sqlparse==0.4.4 + +django-debug-toolbar==3.8.1 +psycopg2-binary==2.9.9 +djangorestframework==3.15.1 +drf-yasg==1.21.7 +django-crispy-forms==2.0 +django-formtools==2.4.1 +django-filter==23.5 +requests==2.27.1 +whitenoise==6.5.0 +django-cors-headers==4.1.0 +django-extensions==3.2.3 +django-rest-multiple-models==2.1.3 +django-reversion==5.0.12 +geopandas==0.10.2 +thefuzz==0.20.0 +djangorestframework-simplejwt==5.3.0 +cryptography==42.0.8 +geojson==3.1.0 +minio==7.2.6 +openpyxl==2.5.14 +djangorestframework-csv==3.0.0 +coverage==7.2.0 +MarkupSafe==2.1.5 + gunicorn==19.9.0 -gevent==1.2.2 -psycopg2-binary>=2.7.1,<2.9 -whitenoise>=4.1.2 -django-crispy-forms>=1.6.1 -django-formtools>=2.0 +gevent==1.3.0 django-settings-export>=1.2.1 -djangorestframework>=3.11.2,<3.12 -django-rest-multiple-models==2.1.0 -requests==2.21.0 -minio==7.1.16 -coverage>=4.4.2 -django-filter>=2.0.0,<2.1 -drf-yasg==1.21.7 -django-cors-headers==2.2.0 -django-extensions==2.0.6 -cryptography<=40.0.2 -django-reversion==2.0.13 -openpyxl==2.5.11 lxml==4.6.3 -GDAL>=2.4,<3.0 -deepdiff>=4 urllib3>=1.24,<1.25 -djangorestframework-csv==2.1.0 -geojson==2.4.1 -MarkupSafe>=2.0.1 -djangorestframework-simplejwt==4.4.0 pyjwt>=2.0,<=2.4.0 -thefuzz==0.19.0 -geopandas==0.9.0 -pyproj==3.0.1 diff --git a/app/backend/wells/models.py b/app/backend/wells/models.py index 3c9c2b0b1b..12770ce370 100644 --- a/app/backend/wells/models.py +++ b/app/backend/wells/models.py @@ -1258,7 +1258,6 @@ def longitude(self): "internal_comments":"Staff only comments and information related to the well, and for internal use only, not to be made public.", "land_district_code":"Codes used to identify legal land district used to help identify the property where the well is located. E.g. Alberni, Barclay, Cariboo.", "legal_pid":"A Parcel Identifier or PID is a nine-digit number that uniquely identifies a parcel in the land title register of in BC. The Registrar of Land Titles assigns PID numbers to parcels for which a title is being entered in the land title register as a registered title. The Land Title Act refers to the PID as “the permanent parcel identifier”.", - "licenced_status_code":"Valid licensing options granted to a well under the Water Sustainability Act. This information comes from eLicensing. i.e. Unlicensed, Licensed, Historical.", "liner_diameter":"Diameter of the liner placed inside the well. Measured in inches.", "liner_from":"Depth below ground level at which the liner starts inside the well. Measured in feet.", "liner_material_code":"Code that describes the material noted for lithology. E.g. Rock, Clay, Sand, Unspecified,", diff --git a/app/backend/wells/signals.py b/app/backend/wells/signals.py index 0925ddb72a..c8902ad4bb 100644 --- a/app/backend/wells/signals.py +++ b/app/backend/wells/signals.py @@ -9,17 +9,54 @@ calculate_score_address, calculate_score_city, calculate_natural_resource_region_for_well, \ reverse_geocode +def _get_utm_zone(geom): + if not geom: + return + return math.floor((geom.x + 180) / 6) + 1 + +def _generate_utm_point(utm_zone, geom): + from osgeo import ogr, osr + + if utm_zone is None: + return + + source_srs = osr.SpatialReference() + source_srs.ImportFromEPSG(4326) + + target_srs = osr.SpatialReference() + target_srs.ImportFromEPSG(32600 + utm_zone) + + transform = osr.CoordinateTransformation(source_srs, target_srs) + + point = ogr.Geometry(ogr.wkbPoint) + + point.AddPoint(geom.y, geom.x) + + point.Transform(transform) + + return point + + @receiver(pre_save, sender=Well) def update_utm(sender, instance, **kwargs): - if instance.geom and (-180 < instance.geom.x < 180): # only update utm when geom is valid - utm_zone = math.floor((instance.geom.x + 180) / 6) + 1 - coord_transform = CoordTransform(SpatialReference(4326), SpatialReference(32600 + utm_zone)) - utm_point = instance.geom.transform(coord_transform, clone=True) - - instance.utm_zone_code = utm_zone - # We round to integers because easting/northing is only precise to 1m. The DB column is also an integer type. - instance.utm_easting = round(utm_point.x) - instance.utm_northing = round(utm_point.y) + if not instance.geom: + return + + geom = instance.geom + utm_is_valid = -180 < geom.x < 180 + + if not utm_is_valid: + return + + utm_zone = _get_utm_zone(geom) + + utm_point = _generate_utm_point(utm_zone, geom) + + instance.utm_zone_code = utm_zone + + # We round to integers because easting/northing is only precise to 1m. The DB column is also an integer type. + instance.utm_easting = round(utm_point.GetX()) + instance.utm_northing = round(utm_point.GetY()) if not TESTING: @receiver(pre_save, sender=Well) @@ -107,4 +144,4 @@ def set_well_attributes(instance): instance.score_city = calculate_score_city(instance, geocoded_address) # Calculate natural resource region of well - instance.natural_resource_region = calculate_natural_resource_region_for_well(instance) \ No newline at end of file + instance.natural_resource_region = calculate_natural_resource_region_for_well(instance) diff --git a/app/backend/wells/tests/test_signals.py b/app/backend/wells/tests/test_signals.py index ad309d0755..81f4e9a368 100644 --- a/app/backend/wells/tests/test_signals.py +++ b/app/backend/wells/tests/test_signals.py @@ -1,9 +1,33 @@ from django.test import TestCase from django.contrib.gis.geos import Point from wells.models import Well -from wells.signals import update_utm +from wells.signals import _get_utm_zone, _generate_utm_point, update_utm +from osgeo import ogr class TestSignals(TestCase): + def test_get_utm_zone(self): + well = Well(geom=None) + + utm_zone = _get_utm_zone(well.geom) + + self.assertEqual(utm_zone, None) + + def test_get_utm_zone(self): + well = Well(geom="POINT(-122.540000 49.260000)") + + utm_zone = _get_utm_zone(well.geom) + + self.assertEqual(utm_zone, 10) + + def test_generate_utm_point(self): + well = Well(geom="POINT(-122.540000 49.260000)") + + utm_point = _generate_utm_point(10, well.geom) + + self.assertEqual(isinstance(utm_point, ogr.Geometry), True) + self.assertEqual(utm_point.GetX(), 533470.030947186) + self.assertEqual(utm_point.GetY(), 5456461.202321483) + def test_update_utm__no_geom__well_not_updated(self): well = Well(geom=None) @@ -31,6 +55,12 @@ def test_update_utm__geom_longitude_above_valid_range__well_not_updated(self): self.assertEqual(well.utm_easting, None) self.assertEqual(well.utm_northing, None) + """ + I'm writing a few duplicate tests here to confirm transormation. + I don't like using for loops in the tests. + TODO: Ideally we can migrate to pytest and use parameterization for this in the future: + - https://docs.pytest.org/en/7.1.x/example/parametrize.html + """ def test_update_utm__geom_valid__well_updated(self): well = Well(geom="POINT(-122.540000 49.260000)") @@ -39,3 +69,21 @@ def test_update_utm__geom_valid__well_updated(self): self.assertEqual(well.utm_zone_code, 10) self.assertEqual(well.utm_easting, 533470) self.assertEqual(well.utm_northing, 5456461) + + def test_update_utm__geom_valid__well_updated_2(self): + well = Well(geom="POINT(-122.500825 49.208751)") + + update_utm(sender=None, instance=well) + + self.assertEqual(well.utm_zone_code, 10) + self.assertEqual(well.utm_easting, 536358) + self.assertEqual(well.utm_northing, 5450782) + + def test_update_utm__geom_valid__well_updated_3(self): + well = Well(geom="POINT(-123.768875 48.981545)") + + update_utm(sender=None, instance=well) + + self.assertEqual(well.utm_zone_code, 10) + self.assertEqual(well.utm_easting, 443742) + self.assertEqual(well.utm_northing, 5425689) diff --git a/app/scripts/backend-command-script.sh b/app/scripts/backend-command-script.sh index 04b7b97eb5..93cd7e378c 100755 --- a/app/scripts/backend-command-script.sh +++ b/app/scripts/backend-command-script.sh @@ -5,7 +5,6 @@ if [ "$ENVIRONMENT" = "local" ]; then set -x && cd /app/backend && mkdir -p .pip && - curl -s -o get-pip.py https://bootstrap.pypa.io/pip/3.5/get-pip.py && python3 get-pip.py && python3 -m pip install --upgrade pip && python3 -m pip install ptvsd && python3 -m pip install --cache-dir=.pip -r requirements.txt && diff --git a/docker-compose.yml b/docker-compose.yml index 667b7f4063..35decd3686 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -142,7 +142,7 @@ services: platform: linux/x86_64 build: context: ./openshift/docker/backend - dockerfile: Dockerfile.dev + dockerfile: Dockerfile hostname: backend env_file: - path: ./.env.test @@ -152,6 +152,7 @@ services: APP_CONTEXT_ROOT: gwells CSRF_COOKIE_SECURE: "False" CUSTOM_GDAL_GEOS: "False" + DATA_LOADED: "False" DATABASE_NAME: gwells DATABASE_USER: "${DATABASE_USER:-gwells}" DATABASE_PASSWORD: "${DATABASE_PASSWORD:-test1}" diff --git a/openshift/docker/backend/Dockerfile b/openshift/docker/backend/Dockerfile index ecb072bbdd..11846e94fa 100644 --- a/openshift/docker/backend/Dockerfile +++ b/openshift/docker/backend/Dockerfile @@ -1,63 +1,17 @@ -FROM registry.access.redhat.com/rhscl/python-36-rhel7:1 +FROM python:3.7-slim +RUN apt-get -y update -# Switch to root user -# -USER root +RUN apt-get -y install git build-essential libgdal-dev +ENV PATH="/usr/bin/python3:${PATH}" -# External libraries required by Python GIS extensions (e.g. GeoDjango, GeoAlchemy) -# Install and configure GEOS -# -# Note: HTTPS will result in certificate errors, hence the downgrade to HTTP here -# +WORKDIR /app -RUN cd /tmp && wget http://download.osgeo.org/geos/geos-3.7.1.tar.bz2 && \ - tar xjf geos-3.7.1.tar.bz2 && \ - cd geos-3.7.1/ && \ - ./configure --prefix=/usr/local && \ - make && \ - make install && \ - ldconfig && \ - cd /tmp && \ - rm -rf /tmp/geos-3.7.1 /tmp/geos-3.7.1.tar.bz2 +RUN python3 -m pip install 'setuptools<58.0' +RUN python3 -m pip install --upgrade pip +COPY . /app -# Install and configure PROJ.4 -# -# Note: HTTPS will result in certificate errors, hence the downgrade to HTTP here -# - -RUN cd /tmp && wget http://download.osgeo.org/proj/proj-5.2.0.tar.gz && \ - wget http://download.osgeo.org/proj/proj-datumgrid-north-america-1.1.tar.gz && \ - tar xzf proj-5.2.0.tar.gz && \ - cd proj-5.2.0/nad && \ - tar xzf ../../proj-datumgrid-north-america-1.1.tar.gz && \ - cd .. && \ - ./configure --prefix=/usr/local && \ - make \ - && make install && \ - ldconfig && \ - rm -rf /tmp/proj-5.2.0 /tmp/proj-5.2.0.tar.gz /tmp/proj-datumgrid-north-america-1.1.tar.gz - - -# Install and configure GDAL -# (without SFCGAL as we aren't using "CREATE EXTENSION postgis_sfcgal;") -# - -RUN cd /tmp && wget http://download.osgeo.org/gdal/2.4.0/gdal-2.4.0.tar.gz && \ - tar zxvf gdal-2.4.0.tar.gz && cd gdal-2.4.0/ && \ - ./configure --prefix=/usr/local --with-python --with-sfcgal=no && \ - make -j4 && \ - make install && \ - ldconfig && \ - rm -rf /tmp/gdal-2.4.0 /tmp/gdal-2.4.0.tar.gz - - -# Configure GDAL file locations -# -RUN echo "/usr/local/lib/" >> /etc/ld.so.conf && \ - ldconfig - - -USER 1001 +# RUN chmod +x load_fixtures.sh works when i pull the dockerfile into backend but not when dockerfile is with other docker files +RUN chmod +x /app diff --git a/openshift/docker/backend/Dockerfile.dev b/openshift/docker/backend/Dockerfile.dev.old similarity index 100% rename from openshift/docker/backend/Dockerfile.dev rename to openshift/docker/backend/Dockerfile.dev.old diff --git a/openshift/docker/backend/Dockerfile.old.base b/openshift/docker/backend/Dockerfile.old.base new file mode 100644 index 0000000000..ecb072bbdd --- /dev/null +++ b/openshift/docker/backend/Dockerfile.old.base @@ -0,0 +1,63 @@ +FROM registry.access.redhat.com/rhscl/python-36-rhel7:1 + + +# Switch to root user +# +USER root + + +# External libraries required by Python GIS extensions (e.g. GeoDjango, GeoAlchemy) +# Install and configure GEOS +# +# Note: HTTPS will result in certificate errors, hence the downgrade to HTTP here +# + +RUN cd /tmp && wget http://download.osgeo.org/geos/geos-3.7.1.tar.bz2 && \ + tar xjf geos-3.7.1.tar.bz2 && \ + cd geos-3.7.1/ && \ + ./configure --prefix=/usr/local && \ + make && \ + make install && \ + ldconfig && \ + cd /tmp && \ + rm -rf /tmp/geos-3.7.1 /tmp/geos-3.7.1.tar.bz2 + + +# Install and configure PROJ.4 +# +# Note: HTTPS will result in certificate errors, hence the downgrade to HTTP here +# + +RUN cd /tmp && wget http://download.osgeo.org/proj/proj-5.2.0.tar.gz && \ + wget http://download.osgeo.org/proj/proj-datumgrid-north-america-1.1.tar.gz && \ + tar xzf proj-5.2.0.tar.gz && \ + cd proj-5.2.0/nad && \ + tar xzf ../../proj-datumgrid-north-america-1.1.tar.gz && \ + cd .. && \ + ./configure --prefix=/usr/local && \ + make \ + && make install && \ + ldconfig && \ + rm -rf /tmp/proj-5.2.0 /tmp/proj-5.2.0.tar.gz /tmp/proj-datumgrid-north-america-1.1.tar.gz + + +# Install and configure GDAL +# (without SFCGAL as we aren't using "CREATE EXTENSION postgis_sfcgal;") +# + +RUN cd /tmp && wget http://download.osgeo.org/gdal/2.4.0/gdal-2.4.0.tar.gz && \ + tar zxvf gdal-2.4.0.tar.gz && cd gdal-2.4.0/ && \ + ./configure --prefix=/usr/local --with-python --with-sfcgal=no && \ + make -j4 && \ + make install && \ + ldconfig && \ + rm -rf /tmp/gdal-2.4.0 /tmp/gdal-2.4.0.tar.gz + + +# Configure GDAL file locations +# +RUN echo "/usr/local/lib/" >> /etc/ld.so.conf && \ + ldconfig + + +USER 1001