Minor parsing fixes

Discussion chats and topics
Toastify to new UI notifications
Custom names in CoderList

.github/workflows/codeql-analysis.yml

@@ -12,8 +12,9 @@
 name: "CodeQL"
 
 on:
-  schedule:
-    - cron: '37 16 * * 3'
+  push:
+    branches:
+      - master
 
 jobs:
   analyze:

legacy/db.class.php

@@ -70,7 +70,7 @@ function select($table, $fields = '*', $where = '1 = 1')
 
         function escapeString($data)
         {
-            //return mysqli_real_escape_string($this->link, $data);
+            if ($data === null) return null;
             return pg_escape_string($this->link, $data);
         }
 

legacy/helper.php

@@ -219,6 +219,9 @@ function curlexec(&$url, $postfields = NULL, $params = array())
                 if (isset($params["cookie_file"])) {
                     $command .= " -b " . escapeshellarg($params["cookie_file"]) . " -c " . escapeshellarg($params["cookie_file"]);
                 }
+                if (isset($params["curl_args_file"])) {
+                    $command .= " " . file_get_contents($params["curl_args_file"]);
+                }
                 $page = shell_exec($command);
             } else {
                 $page = curl_exec($CID);

legacy/js/countdown.js

@@ -98,7 +98,6 @@ function addValueChange(change)
     for (var key in atable)
     {
         var table = atable[key];
-//        console.log(table);
         var afields = new Array();
         var avalues = new Array();
 
@@ -116,11 +115,9 @@ function addValueChange(change)
             afields.push('`' + field + '`');
             avalues.push(value);
         }
-
 
         var obj = document.getElementById('aAdd_' + table);
         if (obj == null) return;
         obj.href = encodeURI('/?action=query&query=INSERT INTO `' + "clist_" + table + '` (' + afields.join(',') + ') VALUES (' + avalues.join(',') +')');
-//        console.log(obj.href);
     }
 }

legacy/module/codingame.com/index.php

@@ -113,8 +113,8 @@
         }
 
         $contests[] = array(
-            'start_time' => $data['date'] / 1000,
-            'end_time' => $end_time,
+            'start_time' => floor($data['date'] / 1000),
+            'end_time' => floor($end_time),
             'duration' => $duration,
             'title' => $data['title'],
             'url' => $prefix_contest . $data['publicId'],

legacy/module/olympiads.ru/index.php

@@ -257,6 +257,24 @@
         $schedule_page = curlexec($url);
         $schedule_page = $replace_months($schedule_page);
 
+        $standings = array();
+        preg_match_all('#<a[^>]*href="(?P<url>[^"]*standing[^"]*)"[^>]*>[^<]*результат[^<]*</a>#', $schedule_page, $matches);
+        $seen = array();
+        foreach ($matches['url'] as $standings_url) {
+            $standings_url = url_merge($url, $standings_url);
+            if (in_array($standings_url, $seen)) {
+                continue;
+            }
+            $seen[] = $standings_url;
+            $standings_page = curlexec($standings_url);
+            preg_match('#<h[0-9][^>]*>(?P<title>[^<]*)#s', $standings_page, $match);
+            $title = html_entity_decode($match['title']);
+            $standings[] = array(
+                "title" => $title,
+                "url" => $standings_url,
+            );
+        }
+
         preg_match_all('#<h4[^>]*id="[^"]*"[^>]*>(?P<title>[^<]*)</h4>.*?<strong>(?P<date>[^<]*[0-9]{4})\sгода</strong>#s', $schedule_page, $matches, PREG_SET_ORDER);
         foreach ($matches as $match) {
             $title = html_entity_decode($match['title']);
@@ -282,14 +300,31 @@
                 $year--;
             }
             $season = $year . '-' . ($year + 1);
+            $standings_season = $year . '-' . (($year + 1) % 100);
+
+            $standings_url = null;
+            foreach ($standings as $ind => $s) {
+                if (
+                    !preg_match("/[Оо]тборочный/i", $title) ||
+                    !preg_match("/[Зз]аочный/i", $s['title']) ||
+                    strpos($s['url'] . ' ' . $s['title'], $standings_season) === false
+                ) {
+                    continue;
+                }
+                $standings_url = $s['url'];
+                unset($standings[$ind]);
+                break;
+            }
+
 
             $contests[] = array(
                 "start_time" => $start_time,
                 "end_time" => $end_time,
-                "title" => $title . '. ' . $main_title,
+                "title" => "$title. $main_title",
                 "url" => $url,
+                "standings_url" => $standings_url,
                 "host" => $HOST,
-                "key" => $season . '. ' . $title,
+                "key" => "$season. $title",
                 "rid" => $RID,
                 "timezone" => $TIMEZONE,
             );

legacy/module/usaco.org/index.php

@@ -2,18 +2,24 @@
     require_once dirname(__FILE__) . "/../../config.php";
 
     $url = 'http://usaco.org/index.php?page=contests';
-    $page = curlexec($url);
+    $curl_params = ['with_curl' => true, 'curl_args_file' => '/sharedfiles/resource/usaco/curl.args'];
+    $page = curlexec($url, NULL, $curl_params);
 
-    preg_match_all('#<a[^>]*href="(?<url>[^"]*)"[^>]*>(?<name>[^<]*[0-9]{4}[^<]*Results)</a>#', $page, $matches, PREG_SET_ORDER);
     $results = array();
+    preg_match_all('#<a[^>]*href="(?<url>[^"]*result[^"]*)"[^>]*>[^<]*(?<name>[0-9]{4}[^<]*Results)</a>#', $page, $matches, PREG_SET_ORDER);
     foreach ($matches as $match) {
         $k = implode(' ', array_slice(explode(' ', $match['name']), 0, 3));
-        $results[$k] = url_merge($url, $match['url']);
+        $results[strtolower($k)] = url_merge($url, $match['url']);
+    }
+    preg_match_all('#<p>[^<]*(?<name>[0-9]{4}[^<]*)<a[^>]*href="(?<url>[^"]*result[^"]*)"[^>]*>here</a>#', $page, $matches, PREG_SET_ORDER);
+    foreach ($matches as $match) {
+        $k = implode(' ', array_slice(explode(' ', $match['name']), 0, 3));
+        $results[strtolower($k)] = url_merge($url, $match['url']);
     }
 
-    $page = curlexec($URL);
+    $page = curlexec($URL, NULL, $curl_params);
 
-    if (!preg_match('#(\d{4})-(\d{4}) Schedule#', $page, $match)) return;
+    if (!preg_match('#>\s*(\d{4})-(\d{4})[^<]*Schedule#', $page, $match)) return;
     list(, $start_year, $end_year) = $match;
 
     preg_match_all("#(?<start_time>[^\s]+\s\d+)-(?<end_time>(?:[^\s]+\s)?\d+):(?<title>[^<]*)#", $page, $matches, PREG_SET_ORDER);
@@ -57,6 +63,7 @@
             date('Y', strtotime($start_time)) . ' ' . $title,
         );
         foreach ($keys as $k) {
+            $k = strtolower($k);
             if (isset($results[$k])) {
                 $c['standings_url'] = $results[$k];
                 unset($results[$k]);

src/clist/admin.py

@@ -3,7 +3,7 @@
 from django.utils import timezone
 from sql_util.utils import SubqueryCount
 
-from clist.models import Banner, Contest, ContestSeries, Problem, ProblemTag, PromoLink, Promotion, Resource
+from clist.models import Banner, Contest, ContestSeries, Discussion, Problem, ProblemTag, PromoLink, Promotion, Resource
 from pyclist.admin import BaseModelAdmin, admin_register
 from ranking.management.commands.parse_statistic import Command as parse_stat
 from ranking.models import Module, Rating
@@ -249,3 +249,9 @@ class PromotionAdmin(BaseModelAdmin):
 class PromoLinkAdmin(BaseModelAdmin):
     list_display = ['name', 'enable', 'desc', 'url']
     search_fields = ['name']
+
+
+@admin_register(Discussion)
+class DiscussionAdmin(BaseModelAdmin):
+    list_display = ['name', 'resource', 'contest', 'problem', 'what', 'where', 'created', 'modified']
+    search_fields = ['name', 'resource', 'contest', 'problem']

src/clist/api/v2.py

@@ -353,7 +353,7 @@ def dehydrate(self, *args, **kwargs):
                 for k in list(problem.keys()):
                     if k.startswith('_'):
                         problem.pop(k, None)
-                for k in settings.PROBLEM_API_IGNORE_FIELD:
+                for k in settings.PROBLEM_API_IGNORE_FIELDS:
                     problem.pop(k, None)
 
         more_fields = bundle.data['more_fields']

src/clist/models.py

@@ -16,7 +16,8 @@
 import requests
 import xgboost as xgb
 from django.conf import settings
-from django.contrib.contenttypes.fields import GenericRelation
+from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
+from django.contrib.contenttypes.models import ContentType
 from django.contrib.postgres.fields import ArrayField
 from django.db import models, transaction
 from django.db.models import Case, F, Max, Q, When
@@ -522,6 +523,12 @@ class Contest(BaseModel):
 
     event_logs = GenericRelation('logify.EventLog', related_query_name='contest')
     virtual_starts = GenericRelation('ranking.VirtualStart', related_query_name='contest')
+    discussions = GenericRelation(
+        'clist.Discussion',
+        content_type_field='what_type',
+        object_id_field='what_id',
+        related_query_name='what_contest',
+    )
 
     has_submissions = models.BooleanField(default=None, null=True, blank=True, db_index=True)
     has_submissions_tests = models.BooleanField(default=None, null=True, blank=True, db_index=True)
@@ -683,21 +690,20 @@ def month_regex(cls):
         return cls._month_regex
 
     @staticmethod
-    def title_neighbors_(title, deep, viewed):
+    def _title_neighbors(title, deep, viewed):
         viewed.add(title)
         if deep == 0:
             return
 
-        for match in re.finditer(rf'([0-9]+|[A-Z]\b|{Contest.month_regex()})', title):
+        for match in re.finditer(rf'(?P<number>\b[0-9]+\b(?:[\W\S]\b[0-9]+\b)*)|(?P<letter>[A-Z]\b)|(?P<month>{Contest.month_regex()})', title):
             for delta in (-1, 1):
                 base_title = title
-                value = match.group(0)
                 values = []
-                if value.isdigit():
-                    value = str(int(value) + delta)
-                elif len(value) == 1:
+                if value := match.group('number'):
+                    value = re.sub('[0-9]+', lambda x: str(int(x.group()) + delta), value)
+                elif value := match.group('letter'):
                     value = chr(ord(value) + delta)
-                else:
+                elif value := match.group('month'):
                     mformat = '%b' if len(value) == 3 else '%B'
                     index = datetime.strptime(value.title(), mformat).month
                     mformats = ['%b', '%B'] if index == 5 else [mformat]
@@ -714,14 +720,14 @@ def title_neighbors_(title, deep, viewed):
                     new_title = base_title[:match.start()] + value + base_title[match.end():]
                     if new_title in viewed:
                         continue
-                    Contest.title_neighbors_(new_title, deep=deep - 1, viewed=viewed)
+                    Contest._title_neighbors(new_title, deep=deep - 1, viewed=viewed)
 
     def similar_contests(self):
         return similar_contests_queryset(self)
 
     def neighbors(self):
         viewed = set()
-        Contest.title_neighbors_(self.title, deep=1, viewed=viewed)
+        Contest._title_neighbors(self.title, deep=1, viewed=viewed)
 
         qs = None
 
@@ -932,16 +938,18 @@ def problem_rating_update_done(self):
 
     def get_statistics_order(self):
         options = self.info.get('standings', {})
-        contest_fields = self.info.get('fields', []).copy()
+        fields = self.info.get('fields', [])
         resource_standings = self.resource.info.get('standings', {})
         order = copy.copy(options.get('order', resource_standings.get('order')))
         if order:
             for f in order:
-                if f.startswith('addition__') and f.split('__', 1)[1] not in contest_fields:
+                if f.startswith('addition__') and f.split('__', 1)[1] not in fields:
                     order = None
                     break
         if order is None:
             order = ['place_as_int', '-solving']
+        if 'penalty' in fields:
+            order.append('penalty')
         return order
 
     @transaction.atomic
@@ -1046,7 +1054,7 @@ class Problem(BaseModel):
     time = models.DateTimeField(default=None, null=True, blank=True)
     start_time = models.DateTimeField(default=None, null=True, blank=True)
     end_time = models.DateTimeField(default=None, null=True, blank=True)
-    index = models.SmallIntegerField(null=True)
+    index = models.SmallIntegerField(null=True, blank=True)
     key = models.TextField()
     name = models.TextField()
     slug = models.TextField(default=None, null=True, blank=True)
@@ -1070,6 +1078,12 @@ class Problem(BaseModel):
 
     activities = GenericRelation('favorites.Activity', related_query_name='problem')
     notes = GenericRelation('notes.Note', related_query_name='problem')
+    discussions = GenericRelation(
+        'clist.Discussion',
+        content_type_field='what_type',
+        object_id_field='what_id',
+        related_query_name='what_problem',
+    )
 
     objects = BaseManager()
     visible_objects = VisibleProblemManager()
@@ -1158,6 +1172,13 @@ def update_tags(self, tags, replace):
             for tag in old_tags:
                 self.tags.remove(tag)
 
+    @property
+    def full_name(self):
+        short_or_key = self.short or self.key
+        if short_or_key == self.name:
+            return short_or_key
+        return f'{short_or_key}. {self.name}'
+
 
 class ProblemTag(BaseModel):
     name = models.TextField(unique=True, db_index=True, null=False)
@@ -1281,3 +1302,28 @@ def save(self, *args, **kwargs):
 
     def __str__(self):
         return f'{self.name} PromoLink#{self.id}'
+
+
+class Discussion(BaseModel):
+    name = models.CharField(max_length=255)
+    url = models.URLField()
+    resource = models.ForeignKey(Resource, null=True, blank=True, on_delete=models.CASCADE)
+    contest = models.ForeignKey(Contest, null=True, blank=True, on_delete=models.CASCADE)
+    problem = models.ForeignKey(Problem, null=True, blank=True, on_delete=models.CASCADE)
+    what_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, related_name='discussion_what')
+    what_id = models.PositiveIntegerField()
+    what = GenericForeignKey('what_type', 'what_id')
+    where_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, related_name='discussion_where')
+    where_id = models.PositiveIntegerField()
+    where = GenericForeignKey('where_type', 'where_id')
+    info = models.JSONField(default=dict, blank=True)
+    with_problem_discussions = models.BooleanField(default=False)
+
+    def __str__(self):
+        return f'{self.name} Discussion#{self.id}'
+
+    class Meta:
+        unique_together = ('what_type', 'what_id', 'where_type', 'where_id')
+        indexes = [
+            models.Index(fields=['what_type', 'what_id', 'with_problem_discussions']),
+        ]