Upload ty notes to sf (#1602)

* add management command to upload thank you notes

* fix school sync

* parentId is required

* use custom object for thank you note

* remove debug print statements

* higher confidence, add cron

* fix processor call

* school admin layout and filters

* error handling

* add opp id to update (prevent IntegreityError)

* remove unused imports/calls in mapbox MC

* formatting

* remove approx enrollment

* use SF native lat/long fields

* use bulk query (to get all records) + create/mod dates

* remove unneeded comment

api/views.py

@@ -2,7 +2,7 @@
 from django.http import JsonResponse, HttpResponse, Http404
 from django.forms.models import model_to_dict
 from django.db.models import Q
-from rest_framework import viewsets, generics
+from rest_framework import viewsets
 from rest_framework import status
 from rest_framework.decorators import api_view
 from rest_framework.response import Response
@@ -17,6 +17,7 @@
 from .models import ProgressTracker, FeatureFlag, WebviewSettings
 from .serializers import AdopterSerializer, ImageSerializer, DocumentSerializer, ProgressSerializer, CustomizationRequestSerializer
 
+
 class AdopterViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Adopter.objects.all()
     serializer_class = AdopterSerializer
@@ -50,6 +51,7 @@ def get_queryset(self):
             queryset = queryset.filter(account_id=account_id)
         return queryset
 
+
 def sticky_note(request):
     sticky_note = StickyNote.for_site(Site.find_for_request(request))
 
@@ -78,6 +80,7 @@ def footer(request):
         'linkedin_link': footer.linkedin_link,
     })
 
+
 def mapbox(request):
     mapbox = MapBoxDataset.objects.all()
     response = []
@@ -90,6 +93,7 @@ def mapbox(request):
 
     return JsonResponse(response, safe=False)
 
+
 def errata_fields(request):
     '''
     Return a JSON representation of fields from the errata.model.errata static options.
@@ -105,6 +109,7 @@ def errata_fields(request):
 
     return JsonResponse(response, safe=False)
 
+
 def schools(request):
     format = request.GET.get('format', 'json')
     q = request.GET.get('q', False)
@@ -165,6 +170,7 @@ def schools(request):
     else:
         return JsonResponse({'error': 'Invalid format requested.'})
 
+
 def flags(request):
     flag_name_query_string = request.GET.get('flag', False)
 
@@ -178,6 +184,7 @@ def flags(request):
         except FeatureFlag.DoesNotExist:
             raise Http404('Flag does not exist')
 
+
 @api_view(['GET', 'POST'])
 @parser_classes([JSONParser])
 def customize_request(request):

donations/migrations/0009_thankyounote_salesforce_id.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.1.1 on 2025-01-06 23:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("donations", "0008_thankyounote_source"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="thankyounote",
+            name="salesforce_id",
+            field=models.CharField(
+                blank=True, default="", help_text="Not null if uploaded to Salesforce", max_length=255
+            ),
+        ),
+    ]

donations/models.py

@@ -22,6 +22,8 @@ class ThankYouNote(models.Model):
     consent_to_share_or_contact = models.BooleanField(default=False)
     contact_email_address = models.EmailField(blank=True, null=True)
     source = models.CharField(max_length=255, default="", blank=True)
+    salesforce_id = models.CharField(max_length=255, default="", blank=True, help_text="Not null if uploaded to Salesforce")
+
 
 class DonationPopup(models.Model):
     download_image = models.ImageField(null=True, blank=True)

openstax/settings/base.py

@@ -256,10 +256,11 @@
 ########
 
 CRONJOBS = [
-    ('0 2 * * *', 'django.core.management.call_command', ['delete_resource_downloads']),
+    # ('0 2 * * *', 'django.core.management.call_command', ['delete_resource_downloads']),
     ('0 6 * * *', 'django.core.management.call_command', ['update_resource_downloads']),
     ('0 0 8 * *', 'django.core.management.call_command', ['update_schools_and_mapbox']),
     ('0 10 * * *', 'django.core.management.call_command', ['update_partners']),
+    ('0 11 * * *', 'django.core.management.call_command', ['sync_thank_you_notes']),
 ]
 
 if ENVIRONMENT == 'prod':

requirements/base.txt

@@ -22,6 +22,7 @@ mapbox==0.18.1
 Pillow==10.3.0
 psycopg2
 python-dotenv
+rapidfuzz
 requests==2.32.2
 sentry-sdk
 simple-salesforce==1.12.5

salesforce/admin.py

@@ -15,8 +15,8 @@
 
 
 class SchoolAdmin(admin.ModelAdmin):
-    list_display = ['name', 'phone']
-    list_filter = ('key_institutional_partner', 'achieving_the_dream_school', 'hbcu', 'texas_higher_ed')
+    list_display = ['name', 'salesforce_id', 'type', 'current_year_students', 'total_school_enrollment', 'updated']
+    list_filter = ('type', 'location', 'updated')
     search_fields = ['name', ]
 
     def has_add_permission(self, request):

salesforce/management/commands/sync_thank_you_notes.py

@@ -0,0 +1,50 @@
+from django.core.management.base import BaseCommand
+from salesforce.models import School
+from donations.models import ThankYouNote
+from salesforce.salesforce import Salesforce
+from rapidfuzz import process, fuzz, utils
+
+
+class Command(BaseCommand):
+    help = "update thank you note records with SF"
+
+    def handle(self, *args, **options):
+        new_thank_you_notes = ThankYouNote.objects.filter(salesforce_id="")
+
+        # fetch schools to do a fuzzy match on with thank you note manually inputted names
+        school_list = {school.name: school.salesforce_id for school in School.objects.all()}
+
+        with Salesforce() as sf:
+            num_created = 0
+            for note in new_thank_you_notes:
+                account_id = school_list["Find Me A Home"]
+
+                # If note has a school name, see if we can match it and use that account id when creating
+                if note.institution:
+                    school_string = note.institution
+                    filtered_choices = [name for name in school_list.keys() if name.lower().startswith(school_string.lower())]
+                    if filtered_choices:
+                        best_match, score, match_key = process.extractOne(school_string, filtered_choices, scorer=fuzz.partial_ratio, processor=utils.default_process)
+
+                        if score > 99:  # found a good match on school name, use that to populate related school in SF
+                            account_id = school_list[best_match]
+
+                response = sf.Thank_You_Note__c.create(
+                    {'Name': f"{note.first_name} {note.last_name} - {note.created}",
+                     'Message__c': note.thank_you_note,
+                     'First_Name__c': note.first_name,
+                     'Last_Name__c': note.last_name,
+                     'Email_Address__c': note.contact_email_address,
+                     'Institution__c': note.institution,
+                     'Source__c': note.source,
+                     'Consent_to_Share__c': note.consent_to_share_or_contact,
+                     'Submitted_Date__c': note.created.strftime('%Y-%m-%d'),
+                     'Related_Account__c': account_id
+                     }
+                )
+
+                note.salesforce_id = response['id']
+                note.save()
+                num_created += 1
+
+            self.stdout.write(self.style.SUCCESS("{} Salesforce Notes Created.".format(num_created)))

salesforce/management/commands/update_opportunities.py

@@ -44,6 +44,7 @@ def process_results(self, results, delete_stale=False):
                 # don't build records for non-active books
                 if record['Opportunity__r']['Book__r']['Active__c']:
                     opportunity, created = AdoptionOpportunityRecord.objects.update_or_create(
+                        opportunity_id=[record['Opportunity__r']['Book__r']['Id']],
                         account_uuid=uuid.UUID(record['Opportunity__r']['Contact__r']['Accounts_UUID__c']),
                         book_name=record['Opportunity__r']['Book__r']['Name'],
                         defaults={'opportunity_id': record['Id'],

salesforce/management/commands/update_schools.py

@@ -10,128 +10,44 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         with Salesforce() as sf:
-            query = "SELECT Name, Id, Phone, " \
-                      "Website, " \
-                      "Type, " \
-                      "School_Location__c, " \
-                      "K_I_P__c, " \
-                      "Achieving_the_Dream_School__c, " \
-                      "HBCU__c, " \
-                      "Texas_Higher_Ed__c, " \
-                      "Approximate_Enrollment__c, " \
-                      "Pell_Grant_Recipients__c, " \
-                      "Students_Pell_Grant__c, " \
-                      "Students_Current_Year__c, " \
-                      "All_Time_Students2__c, " \
-                      "Total_School_Enrollment__c, " \
-                      "Savings_Current_Year__c, " \
-                      "All_Time_Savings2__c, " \
-                      "BillingStreet, " \
-                      "BillingCity, " \
-                      "BillingState, " \
-                      "BillingPostalCode, " \
-                      "BillingCountry, " \
-                      "Address_Latitude__c, " \
-                      "Address_Longitude__c " \
-                      "FROM Account WHERE All_Time_Savings2__c > 0"
-            response = sf.query_all(query)
-            sf_schools = response['records']
-
-            district_query = "SELECT Name, Id, Phone, " \
-                      "RecordTypeId, " \
-                      "Website, " \
-                      "Type, " \
-                      "School_Location__c, " \
-                      "K_I_P__c, " \
-                      "Achieving_the_Dream_School__c, " \
-                      "HBCU__c, " \
-                      "Texas_Higher_Ed__c, " \
-                      "Approximate_Enrollment__c, " \
-                      "Pell_Grant_Recipients__c, " \
-                      "Students_Pell_Grant__c, " \
-                      "Students_Current_Year__c, " \
-                      "All_Time_Students2__c, " \
-                      "Total_School_Enrollment__c, " \
-                      "Savings_Current_Year__c, " \
-                      "All_Time_Savings2__c, " \
-                      "Adoptions_in_District__c, " \
-                      "BillingStreet, " \
-                      "BillingCity, " \
-                      "BillingState, " \
-                      "BillingPostalCode, " \
-                      "BillingCountry, " \
-                      "Address_Latitude__c, " \
-                      "Address_Longitude__c " \
-                      "FROM Account WHERE RecordTypeId = '012U0000000MdzNIAS' AND K_I_P__c = True"
-            district_response = sf.query_all(district_query)
-            sf_districts = district_response['records']
-            #remove duplicates
-            sf_schools_to_update = [x for x in sf_schools if x not in sf_districts]
+            fetch_results = sf.bulk.Account.query("SELECT Name, Id, Phone, " \
+                                "Website, " \
+                                "Type, " \
+                                "School_Location__c, " \
+                                "Students_Current_Year__c, " \
+                                "Total_School_Enrollment__c, " \
+                                "BillingStreet, " \
+                                "BillingCity, " \
+                                "BillingState, " \
+                                "BillingPostalCode, " \
+                                "BillingCountry, " \
+                                "BillingLatitude, " \
+                                "BillingLongitude " \
+                                "FROM Account", lazy_operation=True)
+            sf_schools = []
+            for list_results in fetch_results:
+                sf_schools.extend(list_results)
 
             updated_schools = 0
             created_schools = 0
-            for sf_district in sf_districts:
-                school, created = School.objects.update_or_create(
-                        salesforce_id=sf_district['Id'],
-                        defaults={'name': sf_district['Name'],
-                                  'phone': sf_district['Phone'],
-                                  'website': sf_district['Website'],
-                                  'type': sf_district['Type'],
-                                  'location': sf_district['School_Location__c'],
-                                  'key_institutional_partner': sf_district['K_I_P__c'],
-                                  'achieving_the_dream_school': sf_district['Achieving_the_Dream_School__c'],
-                                  'hbcu': sf_district['HBCU__c'],
-                                  'texas_higher_ed': sf_district['Texas_Higher_Ed__c'],
-                                  'undergraduate_enrollment': sf_district['Approximate_Enrollment__c'],
-                                  'pell_grant_recipients': sf_district['Pell_Grant_Recipients__c'],
-                                  'percent_students_pell_grant': sf_district['Students_Pell_Grant__c'],
-                                  'current_year_students': sf_district['Students_Current_Year__c'],
-                                  'all_time_students': sf_district['All_Time_Students2__c'],
-                                  'total_school_enrollment': sf_district['Total_School_Enrollment__c'],
-                                  'current_year_savings': sf_district['Savings_Current_Year__c'],
-                                  'all_time_savings': sf_district['All_Time_Savings2__c'],
-                                  'physical_country': sf_district['BillingCountry'],
-                                  'physical_street': sf_district['BillingStreet'],
-                                  'physical_city': sf_district['BillingCity'],
-                                  'physical_state_province': sf_district['BillingState'],
-                                  'physical_zip_postal_code': sf_district['BillingPostalCode'],
-                                  'lat': sf_district['Address_Latitude__c'],
-                                  'long': sf_district['Address_Longitude__c'],
-                                  },
-                    )
-                school.save()
-                if created:
-                    created_schools = created_schools + 1
-                else:
-                    updated_schools = updated_schools + 1
 
-            for sf_school in sf_schools_to_update:
+            for sf_school in sf_schools:
                 school, created = School.objects.update_or_create(
                     salesforce_id=sf_school['Id'],
                     defaults={'name': sf_school['Name'],
                               'phone': sf_school['Phone'],
                               'website': sf_school['Website'],
                               'type': sf_school['Type'],
                               'location': sf_school['School_Location__c'],
-                              'key_institutional_partner': sf_school['K_I_P__c'],
-                              'achieving_the_dream_school': sf_school['Achieving_the_Dream_School__c'],
-                              'hbcu': sf_school['HBCU__c'],
-                              'texas_higher_ed': sf_school['Texas_Higher_Ed__c'],
-                              'undergraduate_enrollment': sf_school['Approximate_Enrollment__c'],
-                              'pell_grant_recipients': sf_school['Pell_Grant_Recipients__c'],
-                              'percent_students_pell_grant': sf_school['Students_Pell_Grant__c'],
                               'current_year_students': sf_school['Students_Current_Year__c'],
-                              'all_time_students': sf_school['All_Time_Students2__c'],
                               'total_school_enrollment': sf_school['Total_School_Enrollment__c'],
-                              'current_year_savings': sf_school['Savings_Current_Year__c'],
-                              'all_time_savings': sf_school['All_Time_Savings2__c'],
                               'physical_country': sf_school['BillingCountry'],
                               'physical_street': sf_school['BillingStreet'],
                               'physical_city': sf_school['BillingCity'],
                               'physical_state_province': sf_school['BillingState'],
                               'physical_zip_postal_code': sf_school['BillingPostalCode'],
-                              'lat': sf_school['Address_Latitude__c'],
-                              'long': sf_school['Address_Longitude__c'],
+                              'lat': sf_school['BillingLatitude'],
+                              'long': sf_school['BillingLongitude'],
                               },
                 )
 
@@ -142,5 +58,6 @@ def handle(self, *args, **options):
                     updated_schools = updated_schools + 1
 
             invalidate_cloudfront_caches('salesforce/schools')
-            response = self.style.SUCCESS("Successfully updated {} schools, created {} schools.".format(updated_schools, created_schools))
+            response = self.style.SUCCESS(
+                "Successfully updated {} schools, created {} schools.".format(updated_schools, created_schools))
         self.stdout.write(response)

salesforce/management/commands/upload_mapbox_schools.py

@@ -1,7 +1,5 @@
-import ast
 from django.core.management.base import BaseCommand
 from salesforce.models import School, MapBoxDataset
-from django.core.files.storage import get_storage_class
 from mapbox import Uploader
 from django.conf import settings
 
@@ -65,8 +63,6 @@ def handle(self, *args, **options):
                     }
                     allfeatures["features"].append(feature)
 
-        file_storage = get_storage_class()()
-
         with tempfile.TemporaryDirectory() as tempdir:
             fname = os.path.join(tempdir, 'schools.geojson')
             with open(fname, 'w') as f:

salesforce/migrations/0113_school_created_school_updated.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.1.1 on 2025-01-08 20:09
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("salesforce", "0112_remove_adoptionopportunityrecord_fall_student_number_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="school",
+            name="created",
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="school",
+            name="updated",
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

salesforce/models.py

@@ -62,6 +62,8 @@ class School(models.Model):
     physical_zip_postal_code = models.CharField(max_length=255, null=True, blank=True)
     long = models.DecimalField(max_digits=8, decimal_places=3, null=True, blank=True)
     lat = models.DecimalField(max_digits=8, decimal_places=3, null=True, blank=True)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
 
     def __str__(self):
         return self.name