diff --git a/.env.sentry.template b/.env.sentry.template index 720db38a..adc82df0 100644 --- a/.env.sentry.template +++ b/.env.sentry.template @@ -11,3 +11,4 @@ SENTRY_CRON_MONITOR_CHECKING_LOGS= SENTRY_CRON_MONITOR_REINDEX= SENTRY_CRON_MONITOR_SET_ACCOUNT_RANK= SENTRY_CRON_MONITOR_UPDATE_AUTO_RATING= +SENTRY_CRON_MONITOR_PARSE_ARCHIVE_PROBLEMS= diff --git a/Dockerfile b/Dockerfile index f91c8cd0..b31fe275 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,9 @@ RUN wget https://github.com/stunnel/static-curl/releases/download/8.6.0-1/curl-l mv /tmp/curl /usr/local/bin/curl && \ rm /tmp/curl.tar.xz +# psql +RUN apt install -y postgresql-client + RUN apt update --fix-missing ENV APPDIR=/usr/src/clist @@ -45,7 +48,7 @@ WORKDIR $APPDIR FROM base as dev ENV DJANGO_ENV_FILE .env.dev RUN apt install -y redis-server -CMD sh -c 'redis-server --daemonize yes; scripts/watchdog.bash "python manage.py rqworker" "*.py"; python manage.py runserver 0.0.0.0:10042' +CMD sh -c 'redis-server --daemonize yes; scripts/watchdog.bash "python manage.py rqworker system default" "*.py"; python manage.py runserver 0.0.0.0:10042' COPY config/ipython_config.py . RUN ipython profile create @@ -71,6 +74,7 @@ RUN mkdir /run/daphne COPY config/redis.conf /etc/redis/redis.conf COPY config/supervisord.conf /etc/supervisord.conf + CMD supervisord -c /etc/supervisord.conf @@ -82,7 +86,50 @@ COPY config/loggly/60-loggly.conf /etc/rsyslog.d/60-loggly.conf ENTRYPOINT /entrypoint.sh -FROM nginx:alpine as nginx +FROM nginx:stable-alpine as nginx +# logrotate RUN apk add --no-cache logrotate COPY config/nginx/logrotate.d/nginx /etc/logrotate.d/nginx RUN chmod 0644 /etc/logrotate.d/nginx +# cron +RUN apk add --no-cache logrotate dcron +COPY config/nginx/cron /etc/cron.d/nginx +RUN chmod 0644 /etc/cron.d/nginx +RUN crontab /etc/cron.d/nginx + +CMD crond && nginx -g "daemon off;" + + +FROM postgres:14.3-alpine as postgres +# pg_repack +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + g++ \ + make \ + musl-dev \ + postgresql-dev \ + git \ + lz4-dev \ + zlib-dev \ + bash \ + util-linux \ + gawk \ + && cd /tmp \ + && git clone https://github.com/reorg/pg_repack.git \ + && cd pg_repack \ + && make \ + && make install \ + && apk del .build-deps \ + && rm -rf /tmp/pg_repack +# numfmt +RUN apk add --no-cache coreutils +# cron +RUN apk add --no-cache dcron +COPY config/postgres/cron /etc/cron.d/postgres +RUN chmod 0644 /etc/cron.d/postgres +RUN crontab /etc/cron.d/postgres +# supervisord +RUN apk add --no-cache supervisor +COPY config/postgres/supervisord.conf /etc/supervisord.conf + +CMD supervisord -c /etc/supervisord.conf diff --git a/config/cron b/config/cron index 0e4d0218..a15cc986 100644 --- a/config/cron +++ b/config/cron @@ -1,18 +1,18 @@ PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin DJANGO_ENV_FILE=.env.prod -20,35,55 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_CALENDAR_UPDATE /usr/src/clist/run-manage.bash update_google_calendars -*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_CREATING_NOTIFICATIONS /usr/src/clist/run-manage.bash notification_to_task -*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SENDING_NOTIFICATIONS /usr/src/clist/run-manage.bash sendout_tasks -*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_PARSING_STATISTICS /usr/src/clist/run-manage.bash parse_statistic -*/3 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_PARSING_ACCOUNTS /usr/src/clist/run-manage.bash parse_accounts_infos -30 * * * * /usr/src/clist/run-manage.bash parse_archive_problems -*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_CHECKING_LOGS /usr/src/clist/run-manage.bash check_logs -*/15 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SET_ACCOUNT_RANK /usr/src/clist/run-manage.bash set_account_rank -*/20 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SET_COUNTRY_RANK /usr/src/clist/run-manage.bash set_country_rank -15 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_UPDATE_AUTO_RATING /usr/src/clist/run-manage.bash update_auto_rating +20,35,55 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_CALENDAR_UPDATE /usr/src/clist/run-manage.bash update_google_calendars +*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_CREATING_NOTIFICATIONS /usr/src/clist/run-manage.bash notification_to_task +*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SENDING_NOTIFICATIONS /usr/src/clist/run-manage.bash sendout_tasks +*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_PARSING_STATISTICS /usr/src/clist/run-manage.bash parse_statistic +*/1 * * * * /usr/src/clist/run-manage.bash parse_live_statistics +*/3 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_PARSING_ACCOUNTS /usr/src/clist/run-manage.bash parse_accounts_infos +30 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_PARSE_ARCHIVE_PROBLEMS /usr/src/clist/run-manage.bash parse_archive_problems +*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_CHECKING_LOGS /usr/src/clist/run-manage.bash check_logs +*/15 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SET_ACCOUNT_RANK /usr/src/clist/run-manage.bash set_account_rank +*/20 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SET_COUNTRY_RANK /usr/src/clist/run-manage.bash set_country_rank +15 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_UPDATE_AUTO_RATING /usr/src/clist/run-manage.bash update_auto_rating -# 32 1 * * wed env MONITOR_NAME=SENTRY_CRON_MONITOR_REINDEX /usr/src/clist/run-manage.bash reindex # # 58 3 14-20 * * [ "$(date '+\%u')" -eq 4 ] && cd $PROJECT_DIR && run-one ./manage.py runscript calculate_account_contests >logs/command/calculate_account_contests.log 2>&1 # 58 4 * * 4 cd $PROJECT_DIR && run-one ./manage.py runscript calculate_coder_n_accounts_and_coder_n_contests >logs/command/calculate_coder_n_accounts_and_coder_n_contests.log 2>&1 # 58 5 * * 4 cd $PROJECT_DIR && run-one ./manage.py runscript calculate_resource_contests >logs/command/calculate_resource_contests.log 2>&1 diff --git a/config/loggly/60-loggly.conf b/config/loggly/60-loggly.conf index 0a27780e..431a6fa8 100644 --- a/config/loggly/60-loggly.conf +++ b/config/loggly/60-loggly.conf @@ -1,14 +1,14 @@ $ModLoad imfile $InputFilePollInterval 5 -$InputFileName /logs/nginx/clist-prod-access.log +$InputFileName /logs/nginx/nginx/clist-prod-access.log $InputFileTag nginx-prod-access: $InputFileStateFile /logs/loggly/nginx-prod-access $InputFileSeverity info $InputFileReadMode 0 $InputRunFileMonitor -$InputFileName /logs/nginx/clist-prod-error.log +$InputFileName /logs/nginx/nginx/clist-prod-error.log $InputFileTag nginx-prod-error: $InputFileStateFile /logs/loggly/nginx-prod-error $InputFileSeverity error diff --git a/config/nginx/cron b/config/nginx/cron new file mode 100644 index 00000000..2ce92484 --- /dev/null +++ b/config/nginx/cron @@ -0,0 +1,2 @@ +0 */6 * * * /usr/sbin/logrotate /etc/logrotate.conf >/var/log/logrotate.log 2>&1 +0 0 */9 * * nginx -s reload >/var/log/nginx-reload.log 2>&1 diff --git a/config/postgres/cron b/config/postgres/cron new file mode 100644 index 00000000..e604aede --- /dev/null +++ b/config/postgres/cron @@ -0,0 +1 @@ +5 0 * * thu /usr/src/clist/scripts/repack-database.bash >/var/log/repack_database.log 2>&1 diff --git a/config/postgres/supervisord.conf b/config/postgres/supervisord.conf new file mode 100644 index 00000000..f8a8cb6a --- /dev/null +++ b/config/postgres/supervisord.conf @@ -0,0 +1,28 @@ +[supervisord] +logfile=/var/log/supervisord.log +loglevel=info +nodaemon=true +logfile_maxbytes=1MB +logfile_backups=5 + +[program:postgres] +command=/usr/local/bin/docker-entrypoint.sh postgres -c max_connections=50 -c checkpoint_timeout=60min -c track_activity_query_size=4096 -c shared_buffers=1GB -c effective_cache_size=3GB -c work_mem=64MB -c maintenance_work_mem=500MB +autostart=true +autorestart=true +stdout_logfile=/var/log/postgres_stdout.log +stderr_logfile=/var/log/postgres_stderr.log +stdout_logfile_maxbytes=10MB +stderr_logfile_maxbytes=10MB +stdout_logfile_backups=5 +stderr_logfile_backups=5 + +[program:cron] +command=/usr/sbin/crond -f +autostart=true +autorestart=true +stdout_logfile=/var/log/cron_stdout.log +stderr_logfile=/var/log/cron_stderr.log +stdout_logfile_maxbytes=10MB +stderr_logfile_maxbytes=10MB +stdout_logfile_backups=5 +stderr_logfile_backups=5 diff --git a/config/supervisord.conf b/config/supervisord.conf index 99476e98..e54cdbc4 100644 --- a/config/supervisord.conf +++ b/config/supervisord.conf @@ -49,9 +49,9 @@ stdout_logfile=logs/redis.log stderr_logfile=logs/redis.err [program:rqworker] -command=python manage.py rqworker +command=python manage.py rqworker system default directory=/usr/src/clist/ -numprocs=3 +numprocs=4 process_name=%(program_name)s%(process_num)d user=root autostart=true diff --git a/docker-compose.yml b/docker-compose.yml index 8ce3aeb6..ec8d2eb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,8 @@ services: context: . target: prod networks: - - clist_network + clist_network: + ipv4_address: 172.42.0.102 volumes: - static_files:/usr/src/clist/staticfiles/ - media_files:/usr/src/clist/mediafiles/ @@ -25,7 +26,8 @@ services: context: . target: dev networks: - - clist_network + clist_network: + ipv4_address: 172.42.0.103 volumes: - ./src/:/usr/src/clist/ - ./legacy/api/:/usr/src/clist/legacy/api/ @@ -50,9 +52,11 @@ services: legacy: build: ./legacy networks: - - clist_network + clist_network: + ipv4_address: 172.42.0.104 volumes: - ./legacy:/usr/src/legacy/ + - shared_files:/sharedfiles/ depends_on: - db secrets: @@ -71,22 +75,17 @@ services: - db restart: unless-stopped db: - image: postgres:14.3-alpine + build: + context: . + target: postgres networks: - clist_network volumes: - postgres_data:/var/lib/postgresql/data/ + - ./src/scripts/:/usr/src/clist/scripts/ + - ./logs/postgres/:/var/log/ env_file: - ./.env.db - command: > - postgres - -c max_connections=50 - -c checkpoint_timeout=60min - -c track_activity_query_size=4096 - -c shared_buffers=1GB - -c effective_cache_size=3GB - -c work_mem=64MB - -c maintenance_work_mem=500MB shm_size: 4GB ports: - ${CLIST_DB_PORT:-5432}:5432 @@ -96,7 +95,8 @@ services: context: . target: nginx networks: - - clist_network + clist_network: + ipv4_address: 172.42.0.101 volumes: - static_files:/staticfiles/ - media_files:/mediafiles/ @@ -104,11 +104,10 @@ services: - ./config/nginx/conf.d:/etc/nginx/conf.d/ - certbot_www:/var/www/certbot/ - certbot_conf:/etc/letsencrypt/ - - ./logs/nginx:/var/log/nginx/ + - ./logs/nginx:/var/log/ ports: - 80:80 - 443:443 - command: "/bin/sh -c 'while :; do logrotate /etc/logrotate.conf; sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'" restart: unless-stopped certbot: image: certbot/certbot:latest @@ -166,6 +165,9 @@ networks: driver_opts: com.docker.network.bridge.name: br-clist name: clist + ipam: + config: + - subnet: 172.42.0.0/24 secrets: db_conf: diff --git a/legacy/helper.php b/legacy/helper.php index 53691a82..b2ba5a34 100755 --- a/legacy/helper.php +++ b/legacy/helper.php @@ -184,6 +184,12 @@ function curlexec(&$url, $postfields = NULL, $params = array()) curl_setopt($CID, CURLOPT_HEADER, true); } + if (isset($params["no_body"])) { + curl_setopt($CID, CURLOPT_NOBODY, true); + } else { + curl_setopt($CID, CURLOPT_NOBODY, false); + } + $cachefile = CACHEDIR . "/" . parse_url($url, PHP_URL_HOST) . "-" . md5(preg_replace("#/?timeMin=[^&]*#", "", $url)) . ".html"; if ($postfields !== NULL) { @@ -217,7 +223,7 @@ function curlexec(&$url, $postfields = NULL, $params = array()) } else { $page = curl_exec($CID); } - if (preg_match('#charset=["\']?([-a-z0-9]+)#i', $page, $match)) + if (preg_match('#charset=["\']?([-a-z0-9]+)#i', $page, $match) && !isset($params['no_convert_charset'])) { $charset = $match[1]; if (!preg_match('#^utf.*8$#i', $charset)) @@ -686,4 +692,47 @@ function pop_item(&$array, $path, $default = null) { unset($last_result[$key]); return $result; } + + function parsed_table($table_html) { + $dom = new DOMDocument(); + $dom->loadHTML($table_html); + + $cols = $dom->getElementsByTagName('th'); + $header = array(); + foreach ($cols as $col) { + $header[] = slugify($col->nodeValue); + } + + $rows = $dom->getElementsByTagName('tr'); + $data = array(); + foreach ($rows as $row) { + $cols = $row->getElementsByTagName('td'); + if ($cols->length == 0) { + continue; + } + if ($cols->length != count($header)) { + continue; + } + $headered_cols = array_combine($header, iterator_to_array($cols)); + $row_data = array(); + foreach ($headered_cols as $field => $col) { + $row_data[$field] = trim($col->nodeValue); + $a = $col->getElementsByTagName('a'); + if ($a->length > 0) { + $row_data[$field . ':url'] = $a[0]->getAttribute('href'); + } + } + $data[] = $row_data; + } + return $data; + } + + function current_season_year() { + $year = date('Y'); + $month = date('n'); + if ($month <= 8) { + return $year - 1; + } + return $year; + } ?> diff --git a/legacy/module/acmp.ru/index.php b/legacy/module/acmp.ru/index.php index 8716abe2..033dd6f9 100644 --- a/legacy/module/acmp.ru/index.php +++ b/legacy/module/acmp.ru/index.php @@ -14,7 +14,10 @@ $url = $change_url; $page = curlexec($url, "period=$year", array('http_header' => array("Content-Type: application/x-www-form-urlencoded", "Referer: $referer"))); - preg_match('#

(?P[^<]*)</h1>#', $page, $match); + if (!preg_match('#<h1>(?P<title>[^<]*)</h1>#', $page, $match)) { + trigger_error("Can't find title", E_USER_WARNING); + break; + } if (strpos($match['title'], "$year") === false) { break; } @@ -32,14 +35,29 @@ $page = str_replace(" ", " ", $page); $page = replace_russian_moths_to_number($page); - preg_match('#<h1>Содержание олимпиады "(?P<title>.*?)"</h1>#', $page, $m); + if (!preg_match('#<h1>Содержание олимпиады "(?P<title>.*?)"</h1>#', $page, $m)) { + trigger_error("Can't find title", E_USER_WARNING); + continue; + } $title = $m['title']; - preg_match('#<b[^>]*>Начало олимпиады:</b>(?P<start_time>[^<]*)<#', $page, $m); + + if (!preg_match('#<b[^>]*>Начало олимпиады:</b>(?P<start_time>[^<]*)<#', $page, $m)) { + trigger_error("Can't find start time", E_USER_WARNING); + continue; + } $start_time = $m['start_time']; $start_time = preg_replace('#\s*г\.\s*#', ' ', $start_time); - preg_match('#<b[^>]*>Продолжительность:</b>(?P<duration>[^<]*)<#', $page, $m); + + if (!preg_match('#<b[^>]*>Продолжительность:</b>(?P<duration>[^<]*)<#', $page, $m)) { + trigger_error("Can't find duration", E_USER_WARNING); + continue; + } $duration = $m['duration']; - preg_match('#id_stage=(?P<id>[0-9]+)#', $url, $m); + + if (!preg_match('#id_stage=(?P<id>[0-9]+)#', $url, $m)) { + trigger_error("Can't find id", E_USER_WARNING); + continue; + } $key = $m['id']; $contests[] = array( diff --git a/legacy/module/algotester.com/index.php b/legacy/module/algotester.com/index.php index 297709d7..f29dc518 100755 --- a/legacy/module/algotester.com/index.php +++ b/legacy/module/algotester.com/index.php @@ -33,8 +33,11 @@ $seen = array(); $tournament_ids = array(); + $tournament_urls = array(); foreach ($scoreboard_urls as $scoreboard_url) { $page = curlexec($scoreboard_url, null, array('no_logmsg' => true)); + preg_match('#<title>(?P<title>[^<]*)#', $page, $match); + preg_match_all('#]*href="(?P[^"]*/Tournament/Display/(?P[0-9]+))[^"]*"#', $page, $matches, PREG_SET_ORDER); foreach ($matches as $match) { @@ -44,6 +47,28 @@ $seen[$match['id']] = true; $tournament_ids[] = $match['id']; } + + preg_match_all('#
  • \s*]*href="(?P/?[^"/]+)"[^>]*>[^<]*\s*
    • #', $page, $matches, PREG_SET_ORDER); + foreach ($matches as $match) { + $url = url_merge($scoreboard_url, $match['href']); + if (in_array($url, $tournament_urls)) { + continue; + } + $tournament_urls[] = $url; + } + } + + foreach ($tournament_urls as $tournament_url) { + $page = curlexec($tournament_url, null, array('no_logmsg' => true)); + preg_match_all('#/TournamentList/(?P[0-9]+)#', $page, $matches, PREG_SET_ORDER); + + foreach ($matches as $match) { + if (isset($seen[$match['id']])) { + continue; + } + $seen[$match['id']] = true; + $tournament_ids[] = $match['id']; + } } if (empty($tournament_ids)) { @@ -51,12 +76,20 @@ return; } - $ids = range(1, 100500); + $seen = array(); + $ids = array_merge($tournament_ids, range(1, 100500)); $n_success = 0; foreach ($ids as $tournament_id) { - if (1.1 * $n_success < $tournament_id - 10) { + if (isset($seen[$tournament_id])) { + continue; + } + $seen[$tournament_id] = true; + + $in_tournament = in_array($tournament_id, $tournament_ids); + if (!$in_tournament && 1.1 * $n_success < $tournament_id - 10) { break; } + $offset = 0; $limit = 100; do { @@ -89,7 +122,7 @@ $standings_url = 'https://algotester.com/en/Contest/ViewScoreboard/'. $c['Id']; } $title = $c['Name']['Text']; - $invisible = in_array($tournament_id, $tournament_ids)? 'false' : 'true'; + $contest = array( 'start_time' => $c['ContestStart'], 'end_time' => $c['ContestEnd'], @@ -99,7 +132,7 @@ 'rid' => $RID, 'timezone' => $TIMEZONE, 'key' => $c['Id'], - 'invisible' => $invisible, + 'invisible' => $in_tournament? "false" : "true", 'standings_url' => $standings_url, ); $contests[] = $contest; diff --git a/legacy/module/azspcs.com/index.php b/legacy/module/azspcs.com/index.php new file mode 100644 index 00000000..572bb735 --- /dev/null +++ b/legacy/module/azspcs.com/index.php @@ -0,0 +1,86 @@ +(?P[^/]*)/index.php)">(?P[^<]*)</a>#', $page, $matches, PREG_SET_ORDER); + + foreach ($matches as $match) { + $url = url_merge($main_url, $match['url']); + $page = curlexec($url); + $key = $match['key']; + $title = $match['title']; + $title = preg_replace('#\s*contest\s*$#i', '', $title); + $title = preg_replace("#\s*Al\s*Zimmermann's\s*#i", '', $title); + + unset($start_time, $end_time); + + if (preg_match('#<a[^>]*href="(?P<url>[^"]*)"[^>]*>[^<]*description[^<]*</a>#i', $page, $match)) { + $description_url = url_merge($url, $match['url']); + $description_page = curlexec($description_url, NULL, ['no_convert_charset' => true, 'no_header' => true]); + } else { + $description_page = $page; + } + + $start_regex = '#(?:start|begin)\w*:(?:\s*|<[^>]*>| )*(?P<start_time>[^<]+[^;])<#i'; + if (preg_match($start_regex, $page, $match) || preg_match($start_regex, $description_page, $match)) { + $start_time = preg_replace('#\s*,\s*#', ' ', $match['start_time']); + $start_time = strtotime($start_time); + } + $end_regex = '#(?:end)\w*:(?:\s*|<[^>]*>| )*(?P<end_time>[^<]+[^;])<#i'; + if (preg_match($end_regex, $page, $match) || preg_match($end_regex, $description_page, $match)) { + $end_time = preg_replace('#\s*,\s*#', ' ', $match['end_time']); + $end_time = strtotime($end_time); + } + + + if (!preg_match('#<a[^>]*href="(?P<url>[^"]*)"[^>]*>[^<]*standings[^<]*</a>#i', $page, $match)) { + preg_match('#<a[^>]*href="(?P<url>[^"]*)"[^>]*>[^<]*final\s*results[^<]*</a>#i', $page, $match); + } + $standings_url = url_merge($url, $match['url']); + + if (!isset($start_time) && !isset($end_time)) { + $standings_page = curlexec($standings_url, NULL, ['no_convert_charset' => true, 'no_header' => true]); + $date_regex = '#(?:\d+-\d+-\d+ \d+:\d+:\d+|\d+ \w+ \d+\s*(?:<[^>]*>|,\s*)?\d+:\d+:\d+|\w+ \d+, \d{4})#i'; + preg_match_all($date_regex, $standings_page, $matches1, PREG_SET_ORDER); + preg_match_all($date_regex, $description_page, $matches2, PREG_SET_ORDER); + $matches = array_merge($matches1, $matches2); + + foreach ($matches as $match) { + $time = $match[0]; + $time = preg_replace('#<[^>]*>#', ' ', $time); + $time = preg_replace('#\s*,\s*#', ' ', $time); + $time = strtotime($time); + if (!$time) { + continue; + } + if (!isset($start_time) || $time < $start_time) { + $start_time = $time; + } + if (!isset($end_time) || $time > $end_time) { + $end_time = $time; + } + } + } else if (!isset($start_time)) { + $start_time = $end_time; + } else if (!isset($end_time)) { + $end_time = $start_time; + } + + $contests[] = array( + "start_time" => $start_time, + "end_time" => $end_time, + "title" => $title, + "url" => $url, + "key" => $key, + "rid" => $RID, + "host" => $HOST, + "standings_url" => $standings_url, + ); + } + // print_r($matches); +?> diff --git a/legacy/module/beecrowd.com.br/index.php b/legacy/module/beecrowd.com.br/index.php index a4b303ce..c73aaef3 100644 --- a/legacy/module/beecrowd.com.br/index.php +++ b/legacy/module/beecrowd.com.br/index.php @@ -30,12 +30,12 @@ preg_match_all('# <tr[^>]*>\s* <td[^>]*>\s*<a[^>]*href="(?P<url>[^"]*)"[^>]*>\s*(?P<key>[0-9]+)\s*</a>\s*</td>\s* - <td[^>]*>.*?</td>\s* - <td[^>]*>.*?</td>\s* - <td[^>]*>\s*<a[^>]*>(?P<title>[^<]*)</a>\s*</td>\s* + <td[^>]*>(?:\s*<[^/][^>]*>)*\s*</td>\s* + <td[^>]*>(?:\s*<[^/][^>]*>)*\s*</td>\s* + <td[^>]*>\s*(?:<[^/][^>]*>\s*)*<a[^>]*>(?P<title>[^<]*)</a>\s*(?:<[^>]*>\s*)*</td>\s* <td[^>]*class="[^"]*date[^"]*"[^>]*>(?P<start>[^<]*)</td>\s* <td[^>]*>(?P<duration>[^<]*)</td>\s* - </tr>#xs', + </tr>#x', $page, $matches, PREG_SET_ORDER, @@ -43,6 +43,9 @@ $nothing = true; foreach ($matches as $c) { + foreach ($c as $k => $v) { + $c[$k] = trim($v); + } $url = url_merge($URL, $c['url']); $title = $c['title']; if (substr($title, -3) == '...') { diff --git a/legacy/module/facebook.com/index.php b/legacy/module/facebook.com/index.php index 002ddad8..09cdd7a4 100755 --- a/legacy/module/facebook.com/index.php +++ b/legacy/module/facebook.com/index.php @@ -7,12 +7,14 @@ function get_ids($page) { global $required_urls; preg_match_all('#<link[^>]*href="(?P<href>[^"]*rsrc[^"]*\.js\b[^"]*)"[^>]*>#', $page, $matches); $urls = $matches['href']; - // preg_match_all('#{"type":"js","src":"(?P<href>[^"]*rsrc[^"]*)"#', $page, $matches); - // foreach ($matches['href'] as $u) { - // $u = str_replace('\/', '/', $u); - // $urls[] = $u; - // } - // $urls = array_unique($urls); + + preg_match_all('#{"type":"js","src":"(?P<href>[^"]*rsrc[^"]*)"#', $page, $matches); + foreach ($matches['href'] as $u) { + $u = str_replace('\/', '/', $u); + $urls[] = $u; + } + + $urls = array_unique($urls); $urls_ = array_fill(0, count($urls), null); $offset = 0; @@ -32,72 +34,43 @@ function get_ids($page) { "CodingCompetitionsContestScoreboardQuery", "CCEScoreboardQuery", ); + $required_ids = array_fill_keys($required_ids, true); foreach ($urls as $u) { - if (DEBUG) { - echo "get id url = $u\n"; - } $url = $u; $p = curlexec($u, null, array('no_logmsg' => true)); + $new_ids = array(); if (preg_match_all('#{id:"(?P<id>[^"]*)"(?:[^{}]*(?:{[^}]*})?)*}#', $p, $matches, PREG_SET_ORDER)) { foreach ($matches as $match) { if (preg_match('#,name:"(?P<name>[^"]*)"#', $match[0], $m)) { $ids[$m['name']] = $match['id']; + $new_ids[] = $m['name']; } } } if (preg_match_all('#__d\("(?P<name>[^_]*)_facebookRelayOperation"[^_]*exports="(?P<id>[^"]*)"#', $p, $matches, PREG_SET_ORDER)) { foreach ($matches as $match) { $ids[$match['name']] = $match['id']; + $new_ids[] = $match['name']; } } - $required_ids_ = array(); - foreach ($required_ids as $id) { - if (!isset($ids[$id])) { - $required_ids_[] = $id; + foreach ($new_ids as $k) { + if (isset($required_ids[$k])) { + unset($required_ids[$k]); } } - if (count($required_ids) != count($required_ids_)) { - $required_urls[$url] = true; - } - $required_ids = $required_ids_; - - if (!count($required_ids)) { + if (empty($required_ids)) { break; } } return $ids; } - $url = "https://www.facebook.com/?_fb_noscript=1"; - $page = curlexec($url); - - if (preg_match('#<form[^>]*action="(?P<url>[^"]*/login/[^"]*)"#', $page, $match)) { - $url = $match['url']; - preg_match_all('#<input[^>]*name="(?P<name>[^"]*)"[^>]*value="(?P<value>[^"]*)"#', $page, $matches, PREG_SET_ORDER); - $data = array(); - foreach ($matches as $match) { - $data[$match['name']] = $match['value']; - } - require_once dirname(__FILE__) . "/secret.php"; - $data['email'] = $FACEBOOK_USERNAME; - $data['pass'] = $FACEBOOK_PASSWORD; - unset($FACEBOOK_EMAIL); - unset($FACEBOOK_USERNAME); - unset($FACEBOOK_PASSWORD); - $page = curlexec($url, $data); - } - - $headers = array( - 'pragma: no-cache', - 'cache-control: no-cache', - 'upgrade-insecure-requests: 1', - 'sec-fetch-site: same-origin', - 'sec-fetch-mode: navigate', - 'sec-fetch-user: ?1', - 'sec-fetch-dest: document', - ); + $headers = json_decode(file_get_contents('sharedfiles/resource/facebook/headers.json')); + $headers = array_map(function($k, $v) { return "$k: $v"; }, array_keys((array)$headers), (array)$headers); + $cookie_file = 'sharedfiles/resource/facebook/cookies.txt'; + $curlexec_params = array('http_header' => $headers, 'with_curl' => true, 'cookie_file' => $cookie_file); unset($year); for (;;) { @@ -108,7 +81,7 @@ function get_ids($page) { if (DEBUG) { echo "url = $url\n"; } - $page = curlexec($url, null, array("http_header" => $headers)); + $page = curlexec($url, null, $curlexec_params); unset($fb_dtsg); if (preg_match('#\["DTSGInitialData",\[\],{"token":"(?P<token>[^"]*)"#', $page, $match)) { @@ -179,7 +152,7 @@ function get_ids($page) { $url = rtrim($URL, '/') . "/$year/{$node['contest_vanity']}"; $scoreboard_url = rtrim($url) . '/scoreboard'; $url_ = $scoreboard_url; - $scoreboard_page = curlexec($url_); + $scoreboard_page = curlexec($url_, NULL, ['with_curl' => true]); $scoreboard_ids = get_ids($scoreboard_page); if ($scoreboard_ids) { $info['_scoreboard_ids'] = $scoreboard_ids; diff --git a/legacy/module/icpc.baylor.edu/index.php b/legacy/module/icpc.baylor.edu/index.php index 75e353e4..bca74287 100755 --- a/legacy/module/icpc.baylor.edu/index.php +++ b/legacy/module/icpc.baylor.edu/index.php @@ -23,7 +23,7 @@ $year = $match['year']; $title = trim($match['title']); - if (!preg_match("#>hosted by(?:[^,<]*,)?\s*(?P<where>[^<]*?)\s*<#i", $page, $match)) { + if (!preg_match("#>hosted by\s+(?:the\s+)(?:[^,<]*,)?\s*(?P<where>[^<]*?)\s*<#i", $page, $match)) { trigger_error('Not found where', E_USER_WARNING); return; } @@ -63,6 +63,9 @@ $title .= ". $where, Egypt"; $timezone = 'Africa/Cairo'; $start_date = str_replace($year, $year + 1, $start_date); + } else if (starts_with($where, 'Kazakhstan')) { + $title .= ". $where"; + $timezone = 'Asia/Almaty'; } else { $title .= ". $where"; $timezone = $TIMEZONE; diff --git a/legacy/module/kattis.com/index.php b/legacy/module/kattis.com/index.php new file mode 100644 index 00000000..252bb089 --- /dev/null +++ b/legacy/module/kattis.com/index.php @@ -0,0 +1,31 @@ +<?php + require_once dirname(__FILE__) . '/../../config.php'; + + $_contests = $contests; + $contests = array(); + + $subdomains = $INFO['update']['subdomains']; + foreach ($subdomains as $subdomain_format) { + $n_skip = 0; + for ($year = current_season_year() + 1, $iter = 0; $n_skip < 3; $year--, $iter++) { + $subdomain = strtr($subdomain_format, array('{YY}' => substr($year, 2, 2), '{YYYY}' => $year)); + $HOST = "$subdomain.kattis.com"; + $URL = "https://$HOST/contests/"; + $n_contests = -count($contests); + include './module/open.kattis.com/index.php'; + $n_contests += count($contests); + $n_skip = $n_contests ? 0 : $n_skip + 1; + if ($iter >= 2 && !isset($_GET['parse_full_list'])) { + break; + } + } + } + + foreach ($contests as $contest) { + if (preg_match('/\b(practice|warmup|test|hidden)\b/i', $contest['title'])) { + continue; + } + $_contests[] = $contest; + } + $contests = $_contests; +?> diff --git a/legacy/module/lightoj.com/index.php b/legacy/module/lightoj.com/index.php index 5dd85332..5e1c366c 100755 --- a/legacy/module/lightoj.com/index.php +++ b/legacy/module/lightoj.com/index.php @@ -16,10 +16,12 @@ } foreach ($response['data']['contests']['data'] as $c) { - if ($c['contestVisibilityStr'] == 'private') { - continue; + $visibility = $c['contestVisibilityStr']; + if ($c['isPasswordProtectedBool']) { + $visibility = 'protected'; } - $title = $c['contestTitleStr'] . ' [' . $c['contestVisibilityStr'] . ', ' . $c['contestTypeStr'] . ', ' . $c['contestParticipationTypeStr'] . ']'; + $title = $c['contestTitleStr'] . ' [' . $visibility . ', ' . $c['contestTypeStr'] . ', ' . $c['contestParticipationTypeStr'] . ']'; + $info = array('parse' => $c); $contests[] = array( 'start_time' => $c['contestStartTimestamp'], 'end_time' => $c['contestEndTimestamp'], @@ -29,6 +31,7 @@ 'rid' => $RID, 'timezone' => $TIMEZONE, 'key' => $c['contestId'], + 'info' => $info, ); } diff --git a/legacy/module/open.kattis.com/index.php b/legacy/module/open.kattis.com/index.php index 2114d5e9..95d51569 100755 --- a/legacy/module/open.kattis.com/index.php +++ b/legacy/module/open.kattis.com/index.php @@ -1,8 +1,12 @@ <?php require_once dirname(__FILE__) . '/../../config.php'; + $host = parse_url($URL, PHP_URL_HOST); + $host_parts = explode('.', $host); + $subdomain = count($host_parts) == 3 && $host_parts[0] != 'open'? $host_parts[0] : false; + $urls = array($URL); - if (isset($_GET['parse_full_list'])) { + if (!$subdomain && isset($_GET['parse_full_list'])) { $url_scheme_host = parse_url($URL, PHP_URL_SCHEME) . "://" . parse_url($URL, PHP_URL_HOST); $urls[] = $url_scheme_host . '/past-contests?user_created=off'; } @@ -14,44 +18,43 @@ 'Ongoing' => 'start_time', 'Upcoming' => 'start_time', 'Past' => 'end_time' - ) as $t => $v + ) as $table_title => $date_field ) { - if (!preg_match("#<h2[^>]*>$t</h2>([^<]*<[^/][^>]*>)*\s*<table[^>]*>.*?</table>#s", $page, $match)) { + if (!preg_match("#<h2[^>]*>$table_title</h2>([^<]*<[^/][^>]*>)*\s*<table[^>]*>.*?</table>#s", $page, $match)) { continue; } - preg_match_all('# - <tr[^>]*>\s* - <td[^>]*>\s* - (?:<[^>]*>\s*)+ - <a[^>]*href="(?P<url>[^"]*)"[^>]*>(?P<title>[^<]*)</a>\s* - (?:</[^>]*>\s*)+ - </td>\s* - (?:<td[^>]*>[0-9:\s]+</td>\s*)?? - <td[^>]*>(?P<duration>[^<]*)</td>\s* - <td[^>]*>(?P<date>[^<]*)</td>\s* - (?:<[^>]*>\s*|<button[^>]*>[^<]*</button>)* - </tr> - #msx', - $match[0], - $matches, - PREG_SET_ORDER - ); - foreach ($matches as $data) { - $key = explode('/', $data['url']); + + $table = parsed_table($match[0]); + + foreach ($table as $data) { + $title = $data['name']; + $title = preg_replace('/[^0-9a-z]*â[^0-9a-z]*/i', ' - ', $title); + + $url = url_merge($clean_url, $data['name:url']); + $key = explode('/', $url); $key = end($key); - $date = trim($data['date']); + if ($subdomain) { + $key = "$subdomain.$key"; + } + + $date = trim($data[str_replace('_', '-', $date_field)]); if (substr_count($date, ' ') == 1) { $date = strtotime($date); $day = 24 * 60 * 60; if ($date + $day < time()) { $date += $day; + } else if ($date_field == 'end_time' && $date > time()) { + $date -= $day; } } + + $duration = preg_replace('#^([0-9]+:[0-9]+):[0-9]+$#', '$1', $data['length']); + $contests[] = array( - $v => $date, - 'title' => $data['title'], - 'duration' => preg_replace('#^([0-9]+:[0-9]+):[0-9]+$#', '$1', $data['duration']), - 'url' => url_merge($clean_url, $data['url']), + $date_field => $date, + 'title' => $title, + 'duration' => $duration, + 'url' => $url, 'host' => $HOST, 'rid' => $RID, 'timezone' => $TIMEZONE, diff --git a/legacy/module/yandex.com_cup/index.php b/legacy/module/yandex.com_cup/index.php index ed6413f3..ad003c83 100755 --- a/legacy/module/yandex.com_cup/index.php +++ b/legacy/module/yandex.com_cup/index.php @@ -1,79 +1,37 @@ <?php require_once dirname(__FILE__) . "/../../config.php"; - - $url = $URL; - + $url = 'https://contest.yandex.com/yacup/schedule'; $page = curlexec($url); - $year = date('Y'); - preg_match_all('#<a[^>]*home-hero__link[^>]*home-hero__link_(?P<type>[a-z]+)[^>]*href="(?P<href>[^"]*)"[^>]*>.*?<[^>]*home-hero__title[^>]*>(?P<name>[^<]*)(</[^>]*>)*</a>#', $page, $matches, PREG_SET_ORDER); + preg_match_all('#<h1[^>]*>(?P<name>[^<]*)</h1>\s*(?P<table><table[^>]*>.*?</table>)#s', $page, $matches, PREG_SET_ORDER); foreach ($matches as $category) { - $url = url_merge($URL, $category['href']); - $page = curlexec($url); + $stages = parsed_table($category['table']); - preg_match_all('#<[^>]*home-stages__date[^>]*>(?<date>[^<]*)<[^>]*>\s*<[^>]*home-stages__title[^>]*>(?<title>[^<]*)<[^>]*>#', $page, $matches, PREG_SET_ORDER); - foreach ($matches as $stage) { - $title = $category['name'] . '. ' . $stage['title']; + foreach ($stages as $stage) { + $start_time = $stage['start-time']; + $start_time = preg_replace('#[\(\)]#', '', $start_time); - $sep = '–'; - $start_time = trim($stage['date']); - if (empty($start_time)) { - continue; - } - $start_time = preg_replace("#[^0-9a-z$sep]+#i", " ", $start_time); - $start_time = preg_replace("#\s*$sep\s*#i", $sep, $start_time); - $end_time = null; - if (strpos($start_time, $sep) !== false) { - list($start_time, $end_time) = explode($sep, $start_time); - $start_time = trim($start_time); - $end_time = trim($end_time); - if (preg_match('#^[0-9]+$#', $end_time)) { - $end_time = preg_replace('#[0-9]+#', $end_time, $start_time); - } - } - if (strpos($start_time, $year) === false) { - $start_time .= " $year"; - } - $start_time = '12:00 ' . $start_time; + $end_time = $stage['end-time']; + $end_time = preg_replace('#[\(\)]#', '', $end_time); - if ($end_time) { - if (strpos($end_time, $year) === false) { - $end_time .= " $year"; - } - $end_time = strtotime($end_time) + 24 * 60 * 60; - } + $duration = $stage['duration']; + $duration = preg_replace('#\s*\(.*?\)$#', '', $duration); + $name = $stage['stage']; + $name = preg_replace('#\s+-\s+.*$#', '', $name); + + $title = $category['name'] . '. ' . $name; + + $year = date('Y', strtotime($start_time)); $month = date('n', strtotime($start_time)); if ($month >= 9) { $season = ($year + 0) . "-" . ($year + 1); } else { $season = ($year - 1) . "-" . ($year + 0); } - - $duration = 0; - if (preg_match('#algorithm#i', $category['name'])) { - if (preg_match('#marathon#i', $stage['title'])) { - $duration = 7 * 24 * 60; // 7 days - } else if (preg_match('#sprint#i', $stage['title'])) { - $duration = 120; // 120 minutes - } else if (preg_match('#final#i', $stage['title'])) { - $duration = 120; - } - } else if (preg_match('#back-?end#i', $category['name'])) { - $duration = 300; - } else if (preg_match('#front-?end#i', $category['name'])) { - $duration = 300; - } else if (preg_match('#analytics#i', $category['name'])) { - $duration = 180; - } else if (preg_match('#mobile#i', $category['name'])) { - if (preg_match('#qualifying#i', $stage['title'])) { - $duration = 120; - } - } - - $key = $category['type'] . ' ' . strtolower($stage['title']) . ' ' . $season; + $key = slugify($category['name']) . ' ' . slugify($name) . ' ' . $season; $contests[] = array( 'start_time' => $start_time, @@ -89,7 +47,92 @@ } } - if ($RID == -1) { - print_r($contests); - } + // $url = $URL; + + // $page = curlexec($url); + // $year = date('Y'); + + // preg_match_all('#<a[^>]*home-hero__link[^>]*home-hero__link_(?P<type>[a-z]+)[^>]*href="(?P<href>[^"]*)"[^>]*>.*?<[^>]*home-hero__title[^>]*>(?P<name>[^<]*)(<[^>/]*>[^<]*)*(</[^>]*>\s*)*</a>#', $page, $matches, PREG_SET_ORDER); + + // foreach ($matches as $category) { + // $url = url_merge($URL, $category['href']); + // $page = curlexec($url); + + // preg_match_all('#<[^>]*home-stages__date[^>]*>(?<date>[^<]*)(?:<em>(?P<tag>[^<]*)</em>)?<[^>]*>\s*<[^>]*home-stages__title[^>]*>(?<title>[^<]*)<[^>]*>#', $page, $matches, PREG_SET_ORDER); + // foreach ($matches as $stage) { + // $title = $category['name'] . '. ' . $stage['title']; + + // $sep = '–'; + // $start_time = trim($stage['date']); + // if (empty($start_time)) { + // continue; + // } + // $start_time = preg_replace("#[^0-9a-z$sep]+#i", " ", $start_time); + // $start_time = preg_replace("#\s*$sep\s*#i", $sep, $start_time); + // $end_time = null; + // if (strpos($start_time, $sep) !== false) { + // list($start_time, $end_time) = explode($sep, $start_time); + // $start_time = trim($start_time); + // $end_time = trim($end_time); + // if (preg_match('#^[0-9]+$#', $end_time)) { + // $end_time = preg_replace('#[0-9]+#', $end_time, $start_time); + // } + // } + // if (strpos($start_time, $year) === false) { + // $start_time .= " $year"; + // } + // $start_time = '12:00 ' . $start_time; + + // if ($end_time) { + // if (strpos($end_time, $year) === false) { + // $end_time .= " $year"; + // } + // $end_time = strtotime($end_time) + 24 * 60 * 60; + // } + + // $month = date('n', strtotime($start_time)); + // if ($month >= 9) { + // $season = ($year + 0) . "-" . ($year + 1); + // } else { + // $season = ($year - 1) . "-" . ($year + 0); + // } + + // $duration = 0; + // if (preg_match('#algorithm#i', $category['name'])) { + // if (preg_match('#marathon#i', $stage['title'])) { + // $duration = 7 * 24 * 60; // 7 days + // } else if (preg_match('#sprint#i', $stage['title'])) { + // $duration = 120; // 120 minutes + // } else if (preg_match('#final#i', $stage['title'])) { + // $duration = 120; + // } + // } else if (preg_match('#back-?end#i', $category['name'])) { + // $duration = 300; + // } else if (preg_match('#front-?end#i', $category['name'])) { + // $duration = 300; + // } else if (preg_match('#analytics#i', $category['name'])) { + // $duration = 180; + // } else if (preg_match('#mobile#i', $category['name'])) { + // if (preg_match('#qualifying#i', $stage['title'])) { + // $duration = 120; + // } + // } else if (preg_match('#juniors#i', $category['name'])) { + // $duration = 120; + // } + + // $key = $category['type'] . ' ' . strtolower($stage['title']) . ' ' . $season; + + // $contests[] = array( + // 'start_time' => $start_time, + // 'end_time' => $end_time, + // 'duration' => $duration, + // 'title' => $title, + // 'url' => $url, + // 'host' => $HOST, + // 'rid' => $RID, + // 'timezone' => $TIMEZONE, + // 'key' => $key, + // ); + // } + // } ?> diff --git a/legacy/update.php b/legacy/update.php index 7b86c98c..8daeb422 100755 --- a/legacy/update.php +++ b/legacy/update.php @@ -91,10 +91,6 @@ preg_match_all($resource['regexp'], $page, $matches, PREG_SET_ORDER); - if (DEBUG) { - print_r($matches); - } - $timezone_offset = timezone_offset_get(new DateTimeZone($resource['timezone']), new DateTime("now")); $registration = NULL; @@ -190,9 +186,10 @@ $updated_resources[$contest['rid']] = true; foreach (array('start_time', 'end_time') as $k) { if (isset($contest[$k]) && !is_numeric($contest[$k]) && $contest[$k]) { - if (!preg_match('/(?:[\-\+][0-9]+:[0-9]+|\s[A-Z]{3,}|Z)$/', $contest[$k]) and !empty($contest['timezone']) and strpos($contest[$k], $contest['timezone']) === false) { + if (!preg_match('/(?:[\-\+][0-9]+:[0-9]+|\s[A-Z]{3,}|Z|UTC\+[0-9]+)$/', $contest[$k]) and !empty($contest['timezone']) and strpos($contest[$k], $contest['timezone']) === false) { $contest[$k] .= " " . $contest['timezone']; } + $contest[$k] = preg_replace('/\bUTC\b([+-][0-9]+)/i', '\1', $contest[$k]); $contest[$k] = preg_replace_callback( '/\s([A-Z]{3,})$/', function ($match) { diff --git a/requirements.txt b/requirements.txt index 1029637e..eba7ec9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ Django==5.1 django-add-default-value==0.10.0 +django-autoslug==1.9.9 django-brotli==0.2.1 django-cache-machine==1.2.0 django-cors-headers==4.3.1 @@ -15,13 +16,13 @@ django-ical==1.9.2 django-ipware==7.0.1 django-jsonify==0.3.0 django-json-widget==2.0.1 -django-ltree==0.5.3 +django-ltree-2==0.1.8 django-oauth-toolkit==2.4.0 django-phonenumber-field==7.3.0 django-pivot==1.10.0 django-print-sql==2018.3.6 django-ratelimit==4.1.0 -django-rq==2.10.2 +django-rq==3.0.0 django-sql-utils==0.7.0 django-static-compress==2.0.0 django-static-fontawesome==6.5.2.0 @@ -108,3 +109,5 @@ geoip2==4.8.0 xgboost==2.1.1 scikit-learn==1.5.1 pandas==2.2.2 +beautifulsoup4==4.12.3 +markdown==3.7 diff --git a/src/clist/admin.py b/src/clist/admin.py index 3bc10e51..99cda1f7 100644 --- a/src/clist/admin.py +++ b/src/clist/admin.py @@ -63,8 +63,8 @@ def parse_statistic(self, request, queryset): 'with_medals', 'related', 'merging_contests', 'series', 'allow_updating_statistics_for_participants', 'set_matched_coders_to_members']}], - ['Timing', {'fields': ['statistics_update_required', 'parsed_time', 'wait_for_successful_update_timing', - 'statistic_timing', 'notification_timing', + ['Timing', {'fields': ['statistics_update_required', 'parsed_time', 'parsed_percentage', + 'wait_for_successful_update_timing', 'statistic_timing', 'notification_timing', 'rating_prediction_timing', 'created', 'modified', 'updated']}], ['Rating', {'fields': ['rating_prediction_hash', 'has_fixed_rating_prediction_field', 'rating_prediction_fields']}], @@ -79,8 +79,9 @@ def parse_statistic(self, request, queryset): actions = [parse_statistic] def get_readonly_fields(self, request, obj=None): - ret = ['auto_updated', 'updated', 'parsed_time', 'wait_for_successful_update_timing', - 'statistic_timing', 'notification_timing', 'rating_prediction_timing'] + ret = ['auto_updated', 'updated', 'parsed_time', 'parsed_percentage', 'wait_for_successful_update_timing', + 'statistic_timing', 'notification_timing', 'rating_prediction_timing', + 'slug', 'title_path'] ret += list(super().get_readonly_fields(request, obj)) return ret @@ -132,7 +133,8 @@ def queryset(self, request, queryset): fieldsets = [ [None, {'fields': ['host', 'short_host', 'enable', 'url', 'profile_url', 'avatar_url', 'problem_url', 'icon', 'n_accounts', 'n_contests']}], - ['Parse information', {'fields': ['regexp', 'path', 'parse_url', 'timezone', 'auto_remove_started']}], + ['Parse information', {'fields': ['regexp', 'path', 'parse_url', 'timezone', 'auto_remove_started', + 'has_inherit_medals_to_related']}], ['Calendar information', {'fields': ['color', 'uid']}], ['Rating information', {'fields': ['has_rating_history', 'has_country_rating', 'avg_rating', 'n_rating_accounts', diff --git a/src/clist/api/v1.py b/src/clist/api/v1.py index 9e959e8c..c5d84af0 100644 --- a/src/clist/api/v1.py +++ b/src/clist/api/v1.py @@ -2,9 +2,9 @@ from tastypie import fields from tastypie.resources import ALL_WITH_RELATIONS -from clist.models import Resource, Contest -from true_coders.models import Filter from clist.api.common import BaseModelResource +from clist.models import Contest, Resource +from true_coders.models import Filter class ResourceResource(BaseModelResource): @@ -17,7 +17,7 @@ class Meta(BaseModelResource.Meta): resource_name = 'resource' filtering = { 'id': ['exact', 'in'], - 'name': ['exact', 'iregex', 'regex', 'in'], + 'name': ['exact', 'in'], } ordering = ['id', 'name', ] diff --git a/src/clist/api/v2.py b/src/clist/api/v2.py index a0559ada..37984800 100644 --- a/src/clist/api/v2.py +++ b/src/clist/api/v2.py @@ -2,7 +2,9 @@ import re import arrow +from django.core.exceptions import FieldDoesNotExist from django.db.models import CharField, IntegerField, JSONField, Value +from django.db.models.constants import LOOKUP_SEP from django.db.models.expressions import F from django.db.models.fields.json import KeyTextTransform from django.db.models.functions import Cast @@ -33,7 +35,31 @@ class Meta(CommmonBaseModuelResource.Meta): def build_filters(self, filters=None, *args, **kwargs): filters = filters or {} filters.pop('total_count', None) - return super().build_filters(filters, *args, **kwargs) + + custom_filters = {} + for filter_expr in list(filters): + filter_bits = filter_expr.split(LOOKUP_SEP) + field_name = filter_bits.pop(0) + if field_name not in self.fields: + continue + try: + django_field_name = self.fields[field_name].attribute + self._meta.object_class._meta.get_field(django_field_name) + continue + except FieldDoesNotExist: + pass + filter_type = filter_bits.pop() if filter_bits else 'exact' + lookup_bits = self.check_filtering(field_name, filter_type, filter_bits) + value = self.filter_value_to_python(filters[filter_expr], field_name, filters, filter_expr, filter_type) + db_field_name = LOOKUP_SEP.join(lookup_bits) + qs_filter = "%s%s%s" % (db_field_name, LOOKUP_SEP, filter_type) + custom_filters[qs_filter] = value + filters.pop(filter_expr) + + filters = super().build_filters(filters, *args, **kwargs) + + filters.update(custom_filters) + return filters def dehydrate(self, *args, **kwargs): bundle = super().dehydrate(*args, **kwargs) @@ -73,8 +99,8 @@ class Meta(BaseModelResource.Meta): filtering = { 'total_count': ['exact'], 'id': ['exact', 'in'], - 'name': ['exact', 'iregex', 'regex', 'in'], - 'short': ['exact', 'iregex', 'regex', 'in'], + 'name': ['exact', 'in'], + 'short': ['exact', 'in'], 'n_accounts': ['exact', 'gt', 'lt', 'gte', 'lte'], 'n_contests': ['exact', 'gt', 'lt', 'gte', 'lte'], } @@ -87,8 +113,7 @@ def dehydrate(self, *args, **kwargs): class ContestResource(BaseModelResource): - resource = fields.CharField('resource__host', - help_text='Unicode string data. Use comma to filter multiple resources') + resource = fields.CharField('resource__host') resource_id = fields.IntegerField('resource_id') host = fields.CharField('host') event = fields.CharField('title') @@ -127,7 +152,7 @@ class Meta(BaseModelResource.Meta): 'end_time__during': ['exact'], 'id': ['exact', 'in'], 'resource_id': ['exact', 'in'], - 'resource': ['exact'], + 'resource': ['exact', 'iregex', 'regex', 'in'], 'host': ['exact', 'iregex', 'regex'], 'event': ['exact', 'iregex', 'regex'], 'start': ['exact', 'gt', 'lt', 'gte', 'lte', 'week_day'], @@ -173,7 +198,6 @@ def build_filters(self, filters=None, *args, **kwargs): upcoming = filters.pop('upcoming', None) filtered = filters.pop('filtered', None) category = filters.pop('category', ['api']) - resource = filters.pop('resource', None) with_problems = filters.pop('with_problems', None) format_time = filters.pop('format_time', None) start_time__during = filters.pop('start_time__during', None) @@ -182,8 +206,6 @@ def build_filters(self, filters=None, *args, **kwargs): if filtered is not None: filters['filtered'] = filtered[-1] filters['category'] = category[-1] - if resource: - filters['resource__host__in'] = ','.join(resource).split(',') if upcoming: filters['upcoming'] = upcoming[-1] if with_problems: @@ -362,21 +384,13 @@ class Meta(BaseModelResource.Meta): 'total_count': ['exact'], 'id': ['exact', 'in'], 'resource_id': ['exact', 'in'], - 'resource': ['exact'], - 'handle': ['exact', 'iregex', 'regex'], + 'resource': ['exact', 'iregex', 'regex', 'in'], + 'handle': ['exact', 'in'], 'rating': ['exact', 'gt', 'lt', 'gte', 'lte', 'isnull'], 'overall_rank': ['exact', 'gt', 'lt', 'gte', 'lte', 'isnull'], } ordering = ['id', 'handle', 'rating', 'overall_rank', 'n_contests'] - def build_filters(self, filters=None, *args, **kwargs): - filters = filters or {} - resource = filters.pop('resource', None) - filters = super().build_filters(filters, *args, **kwargs) - if resource: - filters['resource__host'] = resource[-1] - return filters - def apply_filters(self, request, applicable_filters): qs = super().apply_filters(request, applicable_filters) qs = qs.select_related('resource') @@ -430,7 +444,7 @@ class Meta(BaseModelResource.Meta): 'with_accounts': ['exact'], 'country': ['exact'], 'id': ['exact', 'in'], - 'username': ['exact', 'iregex', 'regex', 'in'], + 'username': ['exact', 'in'], } extra_actions = [ { diff --git a/src/clist/api/v3.py b/src/clist/api/v3.py index f3ce1cb6..66bc799f 100644 --- a/src/clist/api/v3.py +++ b/src/clist/api/v3.py @@ -49,7 +49,7 @@ class Meta(BaseModelResource.Meta): 'with_accounts': ['exact'], 'country': ['exact'], 'id': ['exact', 'in'], - 'handle': ['exact', 'iregex', 'regex', 'in'], + 'handle': ['exact', 'in'], 'is_virtual': ['exact'], } extra_actions = [ diff --git a/src/clist/api/v4.py b/src/clist/api/v4.py index d386d33b..fdef86f7 100644 --- a/src/clist/api/v4.py +++ b/src/clist/api/v4.py @@ -10,7 +10,6 @@ from tastypie import fields from clist.api import v3 -from clist.api.common import is_true_value from clist.api.v3 import (BaseModelResource, ContestResource, ResourceResource, StatisticsResource, # noqa use_for_is_real, use_for_is_virtual, use_in_detail_only, use_in_me_only) from clist.models import Contest, Problem, ProblemVerdict @@ -34,8 +33,8 @@ class Meta(v3.AccountResource.Meta): 'total_count': ['exact'], 'id': ['exact', 'in'], 'resource_id': ['exact', 'in'], - 'resource': ['exact'], - 'handle': ['exact', 'iregex', 'regex'], + 'resource': ['exact', 'regex', 'iregex', 'in'], + 'handle': ['exact', 'in'], 'rating': ['exact', 'gt', 'lt', 'gte', 'lte', 'isnull'], 'resource_rank': ['exact', 'gt', 'lt', 'gte', 'lte', 'isnull'], 'last_activity': ['exact', 'gt', 'lt', 'gte', 'lte', 'week_day'], @@ -69,11 +68,11 @@ class Meta(v3.CoderResource.Meta): class ProblemResource(BaseModelResource): name = fields.CharField('name') - contests_ids = fields.ListField('contests_ids', null=True) + contest_ids = fields.ListField('contest_ids', null=True, + help_text="A list of data. Ex: {'abc', 26.73, 8}") divisions = fields.ListField('divisions', null=True) kinds = fields.ListField('kinds', null=True) - resource = fields.CharField('resource__host', - help_text='Unicode string data. Use comma to filter multiple resources') + resource = fields.CharField('resource__host') resource_id = fields.IntegerField('resource_id') slug = fields.CharField('slug', null=True) short = fields.CharField('short', null=True) @@ -87,8 +86,8 @@ class ProblemResource(BaseModelResource): rating = fields.IntegerField('rating', null=True, help_text='Resource rating') favorite = fields.BooleanField('is_favorite', null=True, help_text='User-marked as favorite') note = fields.CharField('note_text', null=True, help_text='User-specified note') - solved = fields.BooleanField(help_text='Solved in resource system or user-marked as solved') - reject = fields.BooleanField(help_text='Rejected in resource system or user-marked as reject') + solved = fields.BooleanField('solved', help_text='Solved in resource system or user-marked as solved') + reject = fields.BooleanField('reject', help_text='Rejected in resource system or user-marked as reject') system_solved = fields.BooleanField('system_solved', help_text='Solved in resource system') system_reject = fields.BooleanField('system_reject', help_text='Rejected in resource system') user_solved = fields.BooleanField('user_solved', help_text='User-marked as solved') @@ -102,12 +101,12 @@ class Meta(BaseModelResource.Meta): excludes = ('total_count', 'solved', 'reject') filtering = { 'total_count': ['exact'], - 'name': ['exact', 'iregex', 'regex'], - 'contests_ids': ['exact', 'contains'], - 'resource': ['exact', 'iregex', 'regex'], + 'name': ['exact', 'in'], + 'contest_ids': ['exact', 'contains'], + 'resource': ['exact', 'iregex', 'regex', 'in'], 'resource_id': ['exact', 'in'], - 'slug': ['exact', 'regex'], - 'short': ['exact', 'iregex', 'regex'], + 'slug': ['exact', 'in'], + 'short': ['exact', 'in'], 'url': ['exact', 'regex'], 'archive_url': ['exact', 'regex'], 'n_attempts': ['exact', 'gt', 'lt', 'gte', 'lte'], @@ -117,7 +116,7 @@ class Meta(BaseModelResource.Meta): 'n_total': ['exact', 'gt', 'lt', 'gte', 'lte'], 'rating': ['exact', 'gt', 'lt', 'gte', 'lte'], 'favorite': ['exact'], - 'note': ['exact', 'iregex', 'regex'], + 'note': ['exact', 'in'], 'solved': ['exact'], 'reject': ['exact'], 'system_solved': ['exact'], @@ -137,7 +136,7 @@ def get_object_list(self, request): contests = Contest.objects.filter(problem_set__id=OuterRef('pk')) contests = contests.values('problem_set__id').annotate(ids=ArrayAgg('id')).values('ids') - problems = problems.annotate(contests_ids=Subquery(contests, output_field=ArrayField(models.IntegerField()))) + problems = problems.annotate(contest_ids=Subquery(contests, output_field=ArrayField(models.IntegerField()))) content_type = ContentType.objects.get_for_model(Problem) coder = request.user.coder if request.user.is_authenticated else None @@ -156,23 +155,3 @@ def get_object_list(self, request): output_field=models.BooleanField())) return problems - - def build_filters(self, filters=None, *args, **kwargs): - filters = filters or {} - - custom_filters = {} - for k in list(filters.keys()): - if k.startswith('note'): - custom_filters[k.replace('note', 'note_text')] = filters.pop(k)[-1] - elif k.startswith('resource'): - custom_filters[k.replace('resource', 'resource__host')] = filters.pop(k)[-1] - elif k.startswith('favorite'): - custom_filters['is_' + k] = is_true_value(filters.pop(k)[-1]) - elif k in ['solved', 'reject', 'system_solved', 'system_reject', 'user_solved', 'user_todo', 'user_reject']: - custom_filters[k] = is_true_value(filters.pop(k)[-1]) - elif k.startswith('contests_ids'): - custom_filters[k] = filters.pop(k) - - filters = super().build_filters(filters, *args, **kwargs) - filters.update(custom_filters) - return filters diff --git a/src/clist/migrations/0164_edit_contest.py b/src/clist/migrations/0164_edit_contest.py new file mode 100644 index 00000000..97f6e7b1 --- /dev/null +++ b/src/clist/migrations/0164_edit_contest.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1 on 2024-11-17 15:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('clist', '0164_contest_parsed_percentage'), ('clist', '0165_contest_inherit_medals_to_related'), ('clist', '0166_remove_contest_inherit_medals_to_related_and_more'), ('clist', '0167_rename_inherit_medals_to_related_resource_has_inherit_medals_to_related')] + + dependencies = [ + ('clist', '0163_update_resource_problems_fields'), + ] + + operations = [ + migrations.AddField( + model_name='contest', + name='parsed_percentage', + field=models.FloatField(blank=True, null=True), + ), + migrations.AddField( + model_name='resource', + name='has_inherit_medals_to_related', + field=models.BooleanField(default=False), + ), + ] diff --git a/src/clist/models.py b/src/clist/models.py index 374902f1..6936c5f3 100644 --- a/src/clist/models.py +++ b/src/clist/models.py @@ -1,9 +1,11 @@ import calendar +import copy import itertools import logging import math import os import re +from collections import defaultdict from collections.abc import Iterable from datetime import datetime, timedelta, timezone from typing import Optional @@ -16,7 +18,7 @@ from django.conf import settings from django.contrib.contenttypes.fields import GenericRelation from django.contrib.postgres.fields import ArrayField -from django.db import models +from django.db import models, transaction from django.db.models import Case, F, Max, Q, When from django.db.models.expressions import Exists, OuterRef from django.db.models.functions import Cast, Ln @@ -109,6 +111,7 @@ class Resource(BaseModel): statistics_fields = models.JSONField(default=dict, blank=True) skip_for_contests_chart = models.BooleanField(default=False) problem_rating_predictor = models.JSONField(default=dict, blank=True) + has_inherit_medals_to_related = models.BooleanField(default=False) RATING_FIELDS = ( 'old_rating', 'new_rating', 'rating', 'rating_perf', 'performance', 'raw_rating', @@ -389,6 +392,17 @@ def load_problem_rating_predictor(self): model.load_model(model_path) return model + def latest_parsed_contest(self): + return self.contest_set.filter(parsed_time__isnull=False).order_by('-end_time').first() + + @staticmethod + def get(value) -> Optional['Resource']: + if isinstance(value, int) or isinstance(value, str) and value.isdigit(): + return Resource.objects.filter(pk=value).first() + if isinstance(value, str): + return Resource.objects.filter(Q(host=value) | Q(short_host=value)).first() + return None + @property def problems_fields_types(self): return self.problems_fields.get('types', {}) @@ -454,6 +468,7 @@ class Contest(BaseModel): n_statistics = models.IntegerField(null=True, blank=True, db_index=True) n_problems = models.IntegerField(null=True, blank=True, db_index=True) parsed_time = models.DateTimeField(null=True, blank=True) + parsed_percentage = models.FloatField(null=True, blank=True) has_hidden_results = models.BooleanField(null=True, blank=True) related = models.ForeignKey('Contest', null=True, blank=True, on_delete=models.SET_NULL, related_name='related_set') merging_contests = models.ManyToManyField('Contest', blank=True, related_name='merged_set') @@ -545,7 +560,8 @@ def save(self, *args, **kwargs): stats.update(new_global_rating=None, global_rating_change=None) if self.is_over(): - self.with_medals = bool(get_item(self.info, 'standings.medals')) or 'medal' in fields + standings_medals = bool(get_item(self.info, 'standings.medals')) + self.with_medals = standings_medals and not self.has_hidden_results or 'medal' in fields self.with_advance = 'advanced' in fields or '_advance' in fields if not self.kind: @@ -591,6 +607,9 @@ def is_running(self): def is_coming(self): return timezone_now() < self.start_time + def with_virtual_start(self): + return self.duration_in_secs and self.duration != self.full_duration + @property def next_time(self): return self.next_time_to(None) @@ -764,7 +783,7 @@ def standings_per_page(self): def full_problems_list(self): problems = self.info.get('problems') if not problems: - return + return [] if isinstance(problems, dict): division_problems = list(problems.get('division', {}).values()) problems = [] @@ -776,7 +795,7 @@ def full_problems_list(self): def problems_list(self): problems = self.info.get('problems') if not problems: - return + return [] if isinstance(problems, dict): division_problems = list(problems.get('division', {}).values()) problems = [] @@ -869,6 +888,80 @@ def problem_rating_update_done(self): self.problem_rating_update_required = False self.save(update_fields=['problem_rating_update_required']) + def get_statistics_order(self): + options = self.info.get('standings', {}) + contest_fields = self.info.get('fields', []).copy() + resource_standings = self.resource.info.get('standings', {}) + order = copy.copy(options.get('order', resource_standings.get('order'))) + if order: + for f in order: + if f.startswith('addition__') and f.split('__', 1)[1] not in contest_fields: + order = None + break + if order is None: + order = ['place_as_int', '-solving'] + return order + + @transaction.atomic + def inherit_medals(self, other): + if self.with_medals: + raise ValueError('already has medals') + if not other.with_medals: + raise ValueError('other contest has no medals') + if self.n_statistics != other.n_statistics: + raise ValueError('different number of statistics') + + rank_medals = {} + rank_n_medals = defaultdict(int) + seen_teams = set() + for stat in other.statistics_set.filter(addition__medal__isnull=False): + if (team := stat.addition.get('team_id')): + if team in seen_teams: + continue + seen_teams.add(team) + rank = stat.place_as_int + medal = stat.addition['medal'] + if rank in rank_medals and rank_medals[rank] != medal: + raise ValueError('multiple medals for the same place') + rank_medals[rank] = medal + rank_n_medals[rank] += 1 + + for stat in self.statistics_set.all(): + rank = stat.place_as_int + if rank not in rank_medals: + continue + if not rank_n_medals[rank]: + raise ValueError('no medals left for the place') + stat.addition['medal'] = rank_medals[rank] + stat.save(update_fields=['addition']) + rank_n_medals[rank] -= 1 + + if any(rank_n_medals.values()): + raise ValueError('not all medals were used') + + if 'medal' not in self.info.get('fields', []): + self.info['fields'].append('medal') + self.save(update_fields=['info']) + self.with_medals = True + self.save(update_fields=['with_medals']) + + def is_finalized(self): + return ( + self.is_over() and + not self.has_hidden_results and + not self.info.get('_timing_statistic_delta_seconds') + ) + + @staticmethod + def get(value) -> Optional['Contest']: + if isinstance(value, int) or isinstance(value, str) and value.isdigit(): + return Contest.objects.filter(pk=value).first() + if isinstance(value, str): + qs = Contest.objects.filter(Q(title=value) | Q(resource__host=value) | + Q(series__name=value) | Q(series__slug=slug(value))) + return qs.order_by('-end_time').first() + return None + class ContestSeries(BaseModel): name = models.TextField(unique=True, db_index=True, null=False) diff --git a/src/clist/templatetags/extras.py b/src/clist/templatetags/extras.py index f266f6fe..c949faa6 100644 --- a/src/clist/templatetags/extras.py +++ b/src/clist/templatetags/extras.py @@ -8,6 +8,7 @@ from collections.abc import Iterable from copy import deepcopy from datetime import datetime, timedelta +from numbers import Number from sys import float_info from urllib.parse import quote_plus, urlparse @@ -34,6 +35,8 @@ from ipware import get_client_ip from unidecode import unidecode +from utils.urlutils import absolute_url as utils_absolute_url + register = template.Library() @@ -143,30 +146,40 @@ def format_time(time, fmt): @register.filter def hr_timedelta(delta, n_significant=2): - if isinstance(delta, timedelta): + if isinstance(delta, datetime): + delta = (delta - now()).total_seconds() + elif isinstance(delta, timedelta): delta = delta.total_seconds() if delta <= 0: return 'past' - ret = [] - n_used = 0 - for c, s in ( + units = [ (364 * 24 * 60 * 60, 'year'), (7 * 24 * 60 * 60, 'week'), (24 * 60 * 60, 'day'), (60 * 60, 'hour'), (60, 'minute'), (1, 'second'), - ): - if c <= delta: - val = delta // c - delta %= c - ret.append('%d %s%s' % (val, s, 's' if val > 1 else '')) - n_used += 1 - elif ret: - n_used += 1 - if n_significant and n_significant == n_used: + ] + + rounded_delta = 0 + for seconds_per_unit, unit_name in units: + if delta >= seconds_per_unit: + n_significant -= 1 + val = round(delta / seconds_per_unit) if n_significant == 0 else delta // seconds_per_unit + delta %= seconds_per_unit + rounded_delta += val * seconds_per_unit + elif rounded_delta: + n_significant -= 1 + if n_significant == 0: break + + ret = [] + for seconds_per_unit, unit_name in units: + if rounded_delta >= seconds_per_unit: + val = rounded_delta // seconds_per_unit + rounded_delta %= seconds_per_unit + ret.append('%d %s%s' % (val, unit_name, 's' if val > 1 else '')) ret = ' '.join(ret) return ret.strip() @@ -281,13 +294,13 @@ def md_escape(value, clear=False): @register.filter -def md_url(value, clear=False): - return value.replace('(', '%28').replace(')', '%29') +def md_url_text(value): + return value.replace('[', '(').replace(']', ')') @register.filter -def md_italic_escape(value): - return value.replace('_', '_\\__') +def md_url(url): + return utils_absolute_url(url.replace('(', '%28').replace(')', '%29')) @register.filter(name='sort') @@ -334,24 +347,38 @@ def calc_mod_penalty(info, contest, solving, penalty): @register.filter -def slug(value): - return slugify(unidecode(value)).strip('-') +def slug(value, sep=None): + ret = slugify(unidecode(value)).strip('-') + if sep: + ret = ret.replace('-', sep) + return ret + + +def is_problems_(contest_or_problems): + return isinstance(contest_or_problems, (list, dict)) + +def get_problems_(contest_or_problems): + if is_problems_(contest_or_problems): + return contest_or_problems + return contest_or_problems.info.get('problems', []) -def get_standings_divisions_order(contest): - problems = contest.info.get('problems', {}) + +def get_standings_divisions_order(contest_or_problems): + problems = get_problems_(contest_or_problems) if 'division' in problems: - divisions_order = list(problems.get('divisions_order', sorted(contest.info['problems']['division'].keys()))) - elif 'divisions_order' in contest.info: - divisions_order = contest.info['divisions_order'] + divisions_order = list(problems.get('divisions_order', sorted(problems['division'].keys()))) + elif not is_problems_(contest_or_problems) and 'divisions_order' in contest_or_problems.info: + divisions_order = contest_or_problems.info['divisions_order'] else: divisions_order = [] return divisions_order @register.filter -def get_division_problems(contest, info): - problems = contest.info.get('problems', []) +def get_division_problems(contest_or_problems, info): + problems = get_problems_(contest_or_problems) + ret = [] seen_keys = set() if 'division' in problems: @@ -360,7 +387,7 @@ def get_division_problems(contest, info): division = info.get('division') if division and division not in divisions: divisions = [division] + divisions - for division in get_standings_divisions_order(contest): + for division in get_standings_divisions_order(contest_or_problems): if division not in divisions: divisions.append(division) for division in divisions: @@ -763,7 +790,7 @@ def divide(value, arg): @register.filter -def substract(value, arg): +def subtract(value, arg): return value - arg @@ -975,6 +1002,25 @@ def is_improved_solution(curr, prev): return False +def time_compare_value(val): + if isinstance(val, Number): + val = [val] + else: + val = list(map(int, val.split(':'))) + return len(val), val + + +def solution_time_compare(lhs, rhs): + for k in 'time_in_seconds', 'time': + if k not in lhs or k not in rhs: + continue + l_val = time_compare_value(lhs[k]) + r_val = time_compare_value(rhs[k]) + if l_val != r_val: + return -1 if l_val < r_val else 1 + return 0 + + @register.filter def timestamp_to_datetime(value): try: @@ -1035,7 +1081,7 @@ def to_dict(**kwargs): @register.filter -def as_number(value, force=False): +def as_number(value, force=False, default=None): valf = str(value).replace(',', '.') retf = asfloat(valf) if retf is not None: @@ -1047,6 +1093,8 @@ def as_number(value, force=False): percentf = asfloat(valf[:-1]) if percentf is not None: return percentf / 100 + if default is not None: + return default if force: return None return value @@ -1199,7 +1247,7 @@ def get_country_from_coder(context, coder): def get_country_from_members(accounts, members): counter = defaultdict(int) for member in members: - if 'account' not in member or member.get('without_country'): + if not member or 'account' not in member: continue account = accounts.get(member['account']) if account is None or not account.country: @@ -1554,6 +1602,13 @@ def get_admin_url(obj): return +@register.simple_tag +def admin_url(obj): + url = get_admin_url(obj) + icon = icon_to('database', '') + return mark_safe(f'<a href="{url}" class="database-link invisible" target="_blank" rel="noopener">{icon}</a>') + + @register.simple_tag def stat_has_failed_verdict(stat, small): return small and not is_solved(stat) and stat.get('verdict') and not stat.get('binary') and not stat.get('icon') @@ -1726,3 +1781,18 @@ def create_dict(**kwargs): @register.simple_tag def create_list(*args): return list(args) + + +@register.filter +def ne(value, arg): + return value != arg + + +@register.simple_tag(takes_context=True) +def absolute_url(context, viewname, *args, **kwargs): + return context['request'].build_absolute_uri(reverse(viewname, args=args, kwargs=kwargs)) + + +@register.filter +def get_id(value): + return id(value) diff --git a/src/clist/views.py b/src/clist/views.py index 2dc04391..7414d1db 100644 --- a/src/clist/views.py +++ b/src/clist/views.py @@ -33,9 +33,8 @@ from ranking.models import Account, CountryAccount, Rating, Statistics from true_coders.models import Coder, CoderList, CoderProblem, Filter, Party from utils.chart import make_bins, make_chart -from utils.custom_request import get_filtered_list from utils.json_field import JSONF -from utils.regex import get_iregex_filter, verify_regex +from utils.regex import get_iregex_filter from utils.timetools import get_timeformat, get_timezone @@ -77,11 +76,16 @@ def get_view_contests(request, coder): resources = [r for r in request.GET.getlist('resource') if r] if resources: base_contests = base_contests.filter(resource_id__in=resources) + search_query = request.GET.get('search_query', None) + if search_query: + base_contests = base_contests.filter(get_iregex_filter(search_query, 'host', 'title')) now = timezone.now() result = [] status = request.GET.get('status') + past_days = int(request.GET.get('past_days', 1)) for group, query, order in ( + ("past", Q(start_time__gt=now - timedelta(days=past_days), end_time__lte=now), "end_time"), ("running", Q(start_time__lte=now, end_time__gte=now), "end_time"), ("coming", Q(start_time__gt=now), "start_time"), ): @@ -152,10 +156,9 @@ def get_events(request): end_time = arrow.get(request.POST.get('end', now + timedelta(days=31))).datetime query = query & Q(end_time__gte=start_time) & Q(start_time__lte=end_time) - search_query = request.POST.get('search_query', None) + search_query = request.POST.get('search_query') if search_query: - search_query_re = verify_regex(search_query) - query &= Q(host__iregex=search_query_re) | Q(title__iregex=search_query_re) + query &= get_iregex_filter(search_query, 'host', 'title') favorite_value = request.POST.get('favorite') if favorite_value == 'on': @@ -193,13 +196,15 @@ def get_events(request): if past_action not in ['show', 'hide'] and contest.end_time < now: color = contest.resource.info.get('get_events', {}).get('colors', {}).get(past_action, color) + start_time = (contest.start_time + timedelta(minutes=offset)).strftime("%Y-%m-%dT%H:%M:%S") + end_time = (contest.end_time + timedelta(minutes=offset)).strftime("%Y-%m-%dT%H:%M:%S") c = { 'id': contest.pk, 'title': contest.title, 'host': contest.host, 'url': contest.actual_url, - 'start': (contest.start_time + timedelta(minutes=offset)).strftime("%Y-%m-%dT%H:%M:%S"), - 'end': (contest.end_time + timedelta(minutes=offset)).strftime("%Y-%m-%dT%H:%M:%S"), + 'start': start_time, + 'end': end_time, 'countdown': contest.next_time_to(now), 'hr_duration': contest.hr_duration, 'color': color, @@ -210,25 +215,35 @@ def get_events(request): c['favorite'] = contest.is_favorite result.append(c) except Exception as e: - return JsonResponse({'message': f'query = `{search_query}`, error = {e}'}, safe=False, status=400) + return JsonResponse({'error': str(e)}, safe=False, status=400) return JsonResponse(result, safe=False) @login_required def send_event_notification(request): - method = request.POST['method'] + methods = request.POST.getlist('method') + methods = set(methods) contest_id = request.POST['contest_id'] message = request.POST['message'] coder = request.user.coder - sendout_tasks.Command().send_message( - coder=coder, - method=method, - data={'contests': [int(contest_id)]}, - message=message.strip() + '\n', - ) - + if len(methods) > 10: + return HttpResponseBadRequest('Too many methods') + + notifications = coder.get_notifications() + notifications = {k for k, v in notifications} + for method in methods: + if method not in notifications: + return HttpResponseBadRequest('Invalid method') + + for method in methods: + sendout_tasks.Command().send_message( + coder=coder, + method=method, + data={'contests': [int(contest_id)]}, + message=message.strip() + '\n', + ) return HttpResponse('ok') @@ -316,6 +331,9 @@ def main(request, party=None): offset = get_timezone_offset(tzname) + more_fields = request.user.has_perm('clist.view_more_fields') + more_fields = more_fields and [f for f in request.GET.getlist('more') if f] or [] + context.update({ "offset": offset, "now": now, @@ -328,6 +346,7 @@ def main(request, party=None): "add_to_calendar": get_add_to_calendar(request), "banners": banners, "promotion": promotion, + "more_fields": more_fields, }) return render(request, "main.html", context) @@ -1152,7 +1171,7 @@ def problems(request, template='problems.html'): field_types = fields_types.get(field) if not field_types or any(t in field_types for t in ('int', 'float', 'dict')): continue - values_list = get_filtered_list(request, field) + values_list = request.get_filtered_list(field) if values_list: values_filter = Q() for value in values_list: diff --git a/src/logify/admin.py b/src/logify/admin.py index 95020156..b9853f0a 100644 --- a/src/logify/admin.py +++ b/src/logify/admin.py @@ -1,7 +1,7 @@ from django.urls import reverse from django.utils.html import format_html -from logify.models import EventLog, PgStatTuple +from logify.models import EventLog, PgStat from pyclist.admin import BaseModelAdmin, admin_register @@ -19,9 +19,15 @@ def related_object_link(self, obj): related_object_link.short_description = 'Related Object' -@admin_register(PgStatTuple) -class PgStatTupleAdmin(BaseModelAdmin): - list_display = ['id', 'table_name', 'app_name', 'table_len', 'tuple_percent', 'dead_tuple_percent', 'free_percent'] +@admin_register(PgStat) +class PgStatAdmin(BaseModelAdmin): + list_display = ['id', 'table_name', 'app_name', 'pretty_table_size', 'pretty_diff_size', 'table_len', + 'tuple_percent', 'dead_tuple_percent', 'free_percent', + 'last_vacuum', 'last_autovacuum', 'last_analyze', 'last_autoanalyze', + 'created', 'modified'] list_filter = ['app_name'] search_fields = ['table_name'] - ordering = ['-table_len'] + ordering = ['-table_size'] + + def get_readonly_fields(self, request, obj=None): + return [f.name for f in self.model._meta.fields] diff --git a/src/logify/management/commands/update_pgstat.py b/src/logify/management/commands/update_pgstat.py new file mode 100644 index 00000000..4b46199f --- /dev/null +++ b/src/logify/management/commands/update_pgstat.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import re +from logging import getLogger + +import tqdm +from django.core.management.base import BaseCommand +from django.db import connection + +from logify.models import PgStat +from utils.attrdict import AttrDict +from utils.db import dictfetchone, find_app_by_table + + +class Command(BaseCommand): + help = 'Updates the PgStat table' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.logger = getLogger('logify.update_pgstat') + + def add_arguments(self, parser): + parser.add_argument('-n', '--limit', type=int, help='number of tables') + parser.add_argument('-f', '--search', type=str, help='search tables') + parser.add_argument('--reset', action='store_true', help='reset initial_table_size') + + def handle(self, *args, **options): + self.stdout.write(str(options)) + args = AttrDict(options) + + with connection.cursor() as cursor: + cursor.execute("SELECT tablename FROM pg_tables WHERE schemaname='public'") + tables = [row[0] for row in cursor.fetchall()] + if args.limit: + tables = tables[:args.limit] + if args.search: + tables = [table for table in tables if re.search(args.search, table)] + + for table in tqdm.tqdm(tables, desc='tables'): + cursor.execute(f"SELECT * FROM pgstattuple('{table}')") + tuple_stats = dictfetchone(cursor) + + fields = [ + 'pg_total_relation_size(relid) AS table_size', + 'pg_size_pretty(pg_total_relation_size(relid)) AS pretty_table_size', + ''' + CASE + WHEN initial_table_size IS NULL THEN 0 + ELSE pg_total_relation_size(relid) - initial_table_size + END AS diff_size + ''', + ''' + CASE + WHEN initial_table_size IS NULL THEN '0' + ELSE pg_size_pretty(pg_total_relation_size(relid) - initial_table_size) + END AS pretty_diff_size + ''', + 'pg_stat_user_tables.last_vacuum', + 'pg_stat_user_tables.last_autovacuum', + 'pg_stat_user_tables.last_analyze', + 'pg_stat_user_tables.last_autoanalyze', + ] + cursor.execute(f''' + SELECT {', '.join(fields)} + FROM pg_stat_user_tables + LEFT JOIN logify_pgstat ON logify_pgstat.table_name = '{table}' + WHERE relname = '{table}' + ''') + table_stats = dictfetchone(cursor) + + defaults = {'app_name': find_app_by_table(table), **tuple_stats, **table_stats} + + pg_stat, created = PgStat.objects.update_or_create(table_name=table, defaults=defaults) + if pg_stat.initial_table_size is None or args.reset: + pg_stat.initial_table_size = pg_stat.table_size + pg_stat.diff_size = 0 + pg_stat.pretty_diff_size = '0' + pg_stat.save(update_fields=['initial_table_size']) diff --git a/src/logify/management/commands/update_pgstattuple.py b/src/logify/management/commands/update_pgstattuple.py deleted file mode 100644 index 56355d50..00000000 --- a/src/logify/management/commands/update_pgstattuple.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 - -import re -from logging import getLogger - -import tqdm -from django.core.management.base import BaseCommand -from django.db import connection - -from logify.models import PgStatTuple -from utils.attrdict import AttrDict -from utils.db import dictfetchone, find_app_by_table - - -class Command(BaseCommand): - help = 'Updates the PgStatTuple table with fresh data from pgstattuple' - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.logger = getLogger('logify.update_pgstattuple') - - def add_arguments(self, parser): - parser.add_argument('-n', '--limit', type=int, help='number of tables') - parser.add_argument('-f', '--search', type=str, help='search tables') - - def handle(self, *args, **options): - self.stdout.write(str(options)) - args = AttrDict(options) - - with connection.cursor() as cursor: - cursor.execute("SELECT tablename FROM pg_tables WHERE schemaname='public'") - tables = [row[0] for row in cursor.fetchall()] - if args.limit: - tables = tables[:args.limit] - if args.search: - tables = [table for table in tables if re.search(args.search, table)] - - for table in tqdm.tqdm(tables, desc='tables'): - cursor.execute(f"SELECT * FROM pgstattuple('{table}')") - stats = dictfetchone(cursor) - defaults = {'app_name': find_app_by_table(table), **stats} - PgStatTuple.objects.update_or_create(table_name=table, defaults=defaults) diff --git a/src/logify/migrations/0010_pretty_pgstat.py b/src/logify/migrations/0010_pretty_pgstat.py new file mode 100644 index 00000000..ccbc4e50 --- /dev/null +++ b/src/logify/migrations/0010_pretty_pgstat.py @@ -0,0 +1,64 @@ +# Generated by Django 5.1 on 2024-11-17 15:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('logify', '0010_rename_pgstattuple_pgstat'), ('logify', '0011_pgstat_last_analyze_pgstat_last_autoanalyze_and_more'), ('logify', '0012_pgstat_table_size'), ('logify', '0013_pgstat_pretty_table_size'), ('logify', '0014_pgstat_diff_size_pgstat_pretty_diff_size'), ('logify', '0015_pgstat_initial_table_size')] + + dependencies = [ + ('logify', '0009_eventlog_environment'), + ] + + operations = [ + migrations.RenameModel( + old_name='PgStatTuple', + new_name='PgStat', + ), + migrations.AddField( + model_name='pgstat', + name='last_analyze', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='last_autoanalyze', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='last_autovacuum', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='last_vacuum', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='table_size', + field=models.BigIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='pretty_table_size', + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='diff_size', + field=models.BigIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='pretty_diff_size', + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AddField( + model_name='pgstat', + name='initial_table_size', + field=models.BigIntegerField(blank=True, null=True), + ), + ] diff --git a/src/logify/models.py b/src/logify/models.py index 7b4c22ff..0f48cf01 100644 --- a/src/logify/models.py +++ b/src/logify/models.py @@ -65,7 +65,7 @@ def update_message(self, message): self.save(update_fields=['message', 'modified', 'elapsed']) -class PgStatTuple(BaseModel): +class PgStat(BaseModel): table_name = models.CharField(max_length=255, db_index=True, unique=True) app_name = models.CharField(max_length=255, blank=True, null=True) table_len = models.BigIntegerField() @@ -78,5 +78,16 @@ class PgStatTuple(BaseModel): free_space = models.BigIntegerField() free_percent = models.FloatField() + last_vacuum = models.DateTimeField(blank=True, null=True) + last_autovacuum = models.DateTimeField(blank=True, null=True) + last_analyze = models.DateTimeField(blank=True, null=True) + last_autoanalyze = models.DateTimeField(blank=True, null=True) + + table_size = models.BigIntegerField(blank=True, null=True) + pretty_table_size = models.CharField(max_length=20, blank=True, null=True) + initial_table_size = models.BigIntegerField(blank=True, null=True) + diff_size = models.BigIntegerField(blank=True, null=True) + pretty_diff_size = models.CharField(max_length=20, blank=True, null=True) + def __str__(self): - return f'{self.table_name} PgStatTuple#{self.id}' + return f'{self.table_name} PgStat#{self.id}' diff --git a/src/my_oauth/admin.py b/src/my_oauth/admin.py index b271beeb..ed64ce20 100644 --- a/src/my_oauth/admin.py +++ b/src/my_oauth/admin.py @@ -1,15 +1,27 @@ +from my_oauth.models import Form, Service, Token from pyclist.admin import BaseModelAdmin, admin_register -from my_oauth.models import Service, Token @admin_register(Service) class ServiceAdmin(BaseModelAdmin): - list_display = ['name', 'title', 'disable'] + list_display = ['name', 'title', '_has_refresh_token', 'disable'] search_fields = ['name', 'title'] + def _has_refresh_token(self, obj): + return bool(obj.refresh_token_uri) + _has_refresh_token.boolean = True + _has_refresh_token.short_description = 'RToken' + @admin_register(Token) class TokenAdmin(BaseModelAdmin): list_display = ['service', 'coder', 'user_id', 'email', 'modified'] - search_fields = ['coder__user__username', 'email'] + search_fields = ['coder__user__username', 'email', 'data'] + list_filter = ['service'] + + +@admin_register(Form) +class FormAdmin(BaseModelAdmin): + list_display = ['name', 'service'] + search_fields = ['name'] list_filter = ['service'] diff --git a/src/my_oauth/migrations/0013_alter_token_email_form.py b/src/my_oauth/migrations/0013_alter_token_email_form.py new file mode 100644 index 00000000..4fec1370 --- /dev/null +++ b/src/my_oauth/migrations/0013_alter_token_email_form.py @@ -0,0 +1,40 @@ +# Generated by Django 5.1 on 2024-10-27 12:44 + +import uuid + +import autoslug.fields +import django.db.models.deletion +from django.db import migrations, models + +import my_oauth.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('my_oauth', '0012_auto_20220211_2316'), + ] + + operations = [ + migrations.AlterField( + model_name='token', + name='email', + field=models.EmailField(blank=True, max_length=254, null=True), + ), + migrations.CreateModel( + name='Form', + fields=[ + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('modified', models.DateTimeField(auto_now=True, db_index=True)), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=255)), + ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)), + ('code', models.TextField()), + ('secret', models.CharField(default=my_oauth.models.generate_secret_64, max_length=64, unique=True)), + ('service', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='my_oauth.service')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/src/my_oauth/migrations/0014_forms.py b/src/my_oauth/migrations/0014_forms.py new file mode 100644 index 00000000..7f22e13c --- /dev/null +++ b/src/my_oauth/migrations/0014_forms.py @@ -0,0 +1,60 @@ +# Generated by Django 5.1 on 2024-11-17 15:34 + +import utils.strings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('my_oauth', '0014_form_register_headers_form_register_url'), ('my_oauth', '0015_remove_form_slug'), ('my_oauth', '0016_alter_form_secret'), ('my_oauth', '0017_token_expires_at'), ('my_oauth', '0018_service_refresh_token_post_service_refresh_token_uri'), ('my_oauth', '0019_form_end_time_form_start_time')] + + dependencies = [ + ('my_oauth', '0013_alter_token_email_form'), + ] + + operations = [ + migrations.AddField( + model_name='form', + name='register_headers', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='form', + name='register_url', + field=models.URLField(blank=True, null=True), + ), + migrations.RemoveField( + model_name='form', + name='slug', + ), + migrations.AlterField( + model_name='form', + name='secret', + field=models.CharField(blank=True, default=utils.strings.generate_secret_64, max_length=64, unique=True), + ), + migrations.AddField( + model_name='token', + name='expires_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='service', + name='refresh_token_post', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='service', + name='refresh_token_uri', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='form', + name='end_time', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='form', + name='start_time', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/src/my_oauth/models.py b/src/my_oauth/models.py index 68b78000..7c484080 100644 --- a/src/my_oauth/models.py +++ b/src/my_oauth/models.py @@ -1,9 +1,12 @@ import re +import uuid from django.db import models +from django.utils import timezone from pyclist.models import BaseManager, BaseModel from true_coders.models import Coder +from utils.strings import generate_secret_64 class ActiveServiceManager(BaseManager): @@ -19,6 +22,8 @@ class Service(BaseModel): code_uri = models.TextField() token_uri = models.TextField() token_post = models.TextField(blank=True) + refresh_token_uri = models.TextField(null=True, blank=True) + refresh_token_post = models.TextField(null=True, blank=True) state_field = models.CharField(max_length=255, default='state') email_field = models.CharField(max_length=255, default='email') user_id_field = models.CharField(max_length=255) @@ -38,8 +43,9 @@ class Token(BaseModel): service = models.ForeignKey(Service, on_delete=models.CASCADE) coder = models.ForeignKey(Coder, null=True, on_delete=models.CASCADE, blank=True) user_id = models.CharField(max_length=255) - email = models.EmailField() + email = models.EmailField(null=True, blank=True) access_token = models.JSONField(default=dict, blank=True) + expires_at = models.DateTimeField(null=True, blank=True) data = models.JSONField(default=dict, blank=True) tokens_view_time = models.DateTimeField(null=True, default=None, blank=True) @@ -61,3 +67,18 @@ def hint(s): return re.sub(regex, '.', s) return f'{hint(login)}@{hint(domain)}' + + +class Form(BaseModel): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=255) + service = models.ForeignKey(Service, on_delete=models.CASCADE) + code = models.TextField() + secret = models.CharField(max_length=64, default=generate_secret_64, blank=True, unique=True) + register_url = models.URLField(null=True, blank=True) + register_headers = models.TextField(null=True, blank=True) + start_time = models.DateTimeField(null=True, blank=True) + end_time = models.DateTimeField(null=True, blank=True) + + def is_closed(self): + return self.end_time is not None and self.end_time < timezone.now() diff --git a/src/my_oauth/urls.py b/src/my_oauth/urls.py index 7403d383..e6aac1b8 100644 --- a/src/my_oauth/urls.py +++ b/src/my_oauth/urls.py @@ -12,4 +12,6 @@ re_path(r'^oauth/([-a-z]+)/$', views.query, name='query'), re_path(r'^oauth/([-a-z]+)/unlink/$', views.unlink, name='unlink'), re_path(r'^oauth/([-a-z]+)/response/$', views.response, name='response'), + re_path(r'^oauth/([-a-z]+)/refresh/$', views.refresh, name='refresh'), + re_path(r'^form/(?P<uuid>[-\w\d]+)/$', views.form, name='form'), ] diff --git a/src/my_oauth/utils.py b/src/my_oauth/utils.py new file mode 100644 index 00000000..b6f97a73 --- /dev/null +++ b/src/my_oauth/utils.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import json +from urllib.parse import parse_qsl + +import requests +from django.forms.models import model_to_dict + + +def access_token_from_response(response): + if response.status_code != requests.codes.ok: + raise Exception('Response status code not equal ok.') + try: + access_token = json.loads(response.text) + except Exception: + access_token = dict(parse_qsl(response.text)) + return access_token + + +def refresh_acccess_token(token): + service = token.service + args = model_to_dict(service) + args.update(token.access_token) + refresh_token_uri = service.refresh_token_uri % args + refresh_token_post = json.loads(service.refresh_token_post % args) + response = requests.post(refresh_token_uri, data=refresh_token_post) + access_token = access_token_from_response(response) + return access_token diff --git a/src/my_oauth/views.py b/src/my_oauth/views.py index 4c75df23..8cbefad8 100644 --- a/src/my_oauth/views.py +++ b/src/my_oauth/views.py @@ -6,7 +6,7 @@ from copy import deepcopy from datetime import timedelta from io import StringIO -from urllib.parse import parse_qsl +from urllib.parse import quote import requests from django.conf import settings @@ -17,14 +17,15 @@ from django.db import transaction from django.db.models import Count, Q from django.forms.models import model_to_dict -from django.http import HttpResponse, HttpResponseBadRequest +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.utils import timezone from flatten_dict import flatten -from clist.templatetags.extras import allowed_redirect, relative_url -from my_oauth.models import Service, Token +from clist.templatetags.extras import allowed_redirect, as_number, relative_url +from my_oauth.models import Form, Service, Token +from my_oauth.utils import access_token_from_response, refresh_acccess_token from true_coders.models import Coder @@ -49,7 +50,19 @@ def unlink(request, name): messages.error(request, 'Not enough services') else: coder.token_set.filter(service__name=name).delete() - return allowed_redirect(reverse('coder:settings', kwargs=dict(tab='social'))) + return allowed_redirect(reverse('coder:settings', kwargs={'tab': 'social'})) + + +@login_required +def refresh(request, name): + token = get_object_or_404(request.user.coder.token_set, + service__name=name, + service__refresh_token_uri__isnull=False) + access_token = refresh_acccess_token(token) + request.session['token_url'] = reverse('coder:settings', kwargs={'tab': 'social'}) + ret = process_access_token(request, token.service, access_token) + request.logger.success(f'Token {token.service.title} refreshed') + return ret def process_data(request, service, access_token, data): @@ -57,28 +70,34 @@ def process_data(request, service, access_token, data): d.update(access_token) d = flatten(d, reducer='underscore') - email = d.get(service.email_field, None) user_id = d.get(service.user_id_field, None) - if not email or not user_id: - raise Exception('Email or User ID not found.') + email = d.get(service.email_field, None) + redirect_url = request.session.pop('token_url', None) + if not user_id: + raise Exception('User ID not found.') + if not email and not redirect_url: + raise Exception('Email not found.') token, created = Token.objects.get_or_create(service=service, user_id=user_id) token.access_token = access_token token.data = data token.email = email + expires_in = as_number(d.get('expires_in'), force=True) + if expires_in: + token.expires_at = timezone.now() + timedelta(seconds=expires_in) token.save() + if redirect_url: + token_id_field = request.session.pop('token_id_field', None) + if token_id_field: + request.session[token_id_field] = token.id + return allowed_redirect(redirect_url) + request.session['token_id'] = token.id return redirect('auth:signup') -def process_access_token(request, service, response): - if response.status_code != requests.codes.ok: - raise Exception('Response status code not equal ok.') - try: - access_token = json.loads(response.text) - except Exception: - access_token = dict(parse_qsl(response.text)) +def process_access_token(request, service, access_token): if service.data_header: args = model_to_dict(service) @@ -132,13 +151,16 @@ def response(request, name): else: url = re.sub('[\n\r]', '', service.token_uri % args) response = requests.get(url) - return process_access_token(request, service, response) + access_token = access_token_from_response(response) + return process_access_token(request, service, access_token) except Exception as e: messages.error(request, "ERROR: {}".format(str(e).strip("'"))) return signup(request) def login(request): + request.session.pop('token_url', None) + redirect_url = request.GET.get('next') if not redirect_url or not redirect_url.startswith('/'): redirect_url = relative_url(request.META.get('HTTP_REFERER')) or reverse('clist:main') @@ -287,3 +309,55 @@ def services_dumpdata(request): service['fields']['secret'] = None service['fields']['app_id'] = None return HttpResponse(json.dumps(services), content_type="application/json") + + +def form(request, uuid): + form = get_object_or_404(Form.objects, pk=uuid) + token_id = request.session.get('form_token_id', None) + token = Token.objects.filter(pk=token_id).first() if token_id else None + + if form.is_closed(): + token = None + code = None + elif token: + data = {k: quote(str(v)) for k, v in flatten(token.data, reducer='underscore').items()} + code = form.code.format(**data) + else: + code = None + + action = request.GET.get('action') + if action: + if form.is_closed(): + return HttpResponseBadRequest('Form is closed') + form_url = reverse('auth:form', args=(uuid, )) + if action == 'login': + request.session['token_id_field'] = 'form_token_id' + request.session['token_url'] = form_url + return redirect(reverse('auth:query', args=(form.service.name, ))) + elif action == 'logout': + request.session.pop('form_token_id', None) + elif action == 'register': + if request.headers.get('X-Secret') != form.secret: + return HttpResponseBadRequest('Unauthorized') + register_url = form.register_url.format(**request.GET.dict()) + register_headers = form.register_headers.format(**request.headers) + response = requests.post(register_url, headers=json.loads(register_headers)) + return JsonResponse( + { + 'status': 'ok', + 'code': response.status_code, + 'text': response.text, + }, + status=response.status_code, + ) + else: + return HttpResponseBadRequest('Unknown action') + return allowed_redirect(form_url) + + return render(request, 'form.html', { + 'form': form, + 'code': code, + 'token': token, + 'nofavicon': True, + 'nocounter': True, + }) diff --git a/src/notification/admin.py b/src/notification/admin.py index 805bc403..415b8aa5 100644 --- a/src/notification/admin.py +++ b/src/notification/admin.py @@ -1,3 +1,5 @@ +from sql_util.utils import SubqueryCount + from notification.models import Calendar, Notification, NotificationMessage, Subscription, Task from pyclist.admin import BaseModelAdmin, admin_register @@ -8,15 +10,28 @@ class NotificationAdmin(BaseModelAdmin): list_filter = ['method'] search_fields = ['coder__user__username', 'method', 'period'] - def get_readonly_fields(self, request, obj=None): - return super().get_readonly_fields(request, obj) - @admin_register(Subscription) class SubscriptionAdmin(BaseModelAdmin): - list_display = ['coder', 'method', 'resource', 'contest', 'account', 'coder_list', 'coder_chat', 'enable'] + list_display = ['coder', 'method', 'enable', 'n_accounts', 'n_coders', + 'resource', 'contest', 'coder_list', 'coder_chat'] list_filter = ['enable', 'method'] - search_fields = ['coder__username', 'account__key'] + search_fields = ['coder__username', 'accounts__key', 'coders__username'] + + def n_accounts(self, obj): + return obj.n_accounts + + def n_coders(self, obj): + return obj.n_coders + + def get_queryset(self, request): + ret = super().get_queryset(request) + ret = ret.annotate(n_accounts=SubqueryCount('accounts')) + ret = ret.annotate(n_coders=SubqueryCount('coders')) + return ret + + def get_readonly_fields(self, *args, **kwargs): + return ['last_contest', 'last_update'] + super().get_readonly_fields(*args, **kwargs) @admin_register(Task) diff --git a/src/notification/forms.py b/src/notification/forms.py index ede1f51a..10c1d0c2 100644 --- a/src/notification/forms.py +++ b/src/notification/forms.py @@ -29,7 +29,6 @@ class Meta: def __init__(self, coder, *args, **kwargs): super(NotificationForm, self).__init__(*args, **kwargs) - methods = coder.get_notifications() self.fields['method'] = ChoiceField(choices=methods) diff --git a/src/notification/management/commands/sendout_tasks.py b/src/notification/management/commands/sendout_tasks.py index fd1954d2..194b6135 100644 --- a/src/notification/management/commands/sendout_tasks.py +++ b/src/notification/management/commands/sendout_tasks.py @@ -32,6 +32,7 @@ from utils.traceback_with_vars import colored_format_exc logger = getLogger('notification.sendout.tasks') +lock = FileLock('sharedfiles/lock/sendout_tasks.lock') class Command(BaseCommand): @@ -185,6 +186,7 @@ def save_config(self): yaml.dump(self.config, fo, indent=2) @print_sql_decorator() + @lock def handle(self, *args, **options): self.load_config() dryrun = options.get('dryrun') @@ -206,7 +208,11 @@ def handle(self, *args, **options): qs = Task.objects.all() if coders and options.get('force') else Task.unsent.all() if coders: - qs = qs.filter(periodical_notification__coder__username__in=coders) + qs = qs.filter( + Q(periodical_notification__coder__username__in=coders) | + Q(subscription__coder__username__in=coders) + ) + qs = qs.prefetch_related( Prefetch( 'notification__coder__chat_set', diff --git a/src/notification/migrations/0042_edit_subscription.py b/src/notification/migrations/0042_edit_subscription.py new file mode 100644 index 00000000..d592501b --- /dev/null +++ b/src/notification/migrations/0042_edit_subscription.py @@ -0,0 +1,102 @@ +# Generated by Django 5.1 on 2024-11-17 15:40 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('notification', '0042_remove_subscription_notificatio_contest_559967_idx_and_more'), ('notification', '0043_alter_subscription_accounts'), ('notification', '0044_alter_notification_coder_alter_subscription_coder'), ('notification', '0045_alter_notification_coder_alter_subscription_coder'), ('notification', '0046_subscription_coders_alter_subscription_accounts'), ('notification', '0047_subscription_notificatio_enable_2370eb_idx'), ('notification', '0048_subscription_last_contest_subscription_last_update'), ('notification', '0049_subscription_top_n_subscription_with_first_accepted'), ('notification', '0050_subscription_notificatio_resourc_b8cd18_idx_and_more')] + + dependencies = [ + ('clist', '0163_update_resource_problems_fields'), + ('clist', '0167_rename_inherit_medals_to_related_resource_has_inherit_medals_to_related'), + ('notification', '0041_alter_subscription_account_and_more'), + ('ranking', '0132_account_rating_prediction'), + ('ranking', '0135_alter_account_n_subscriptions'), + ('ranking', '0136_rename_n_subscriptions_account_n_subscribers'), + ('ranking', '0140_parsestatistics_parse_time'), + ('tg', '0013_chat_accounts'), + ('true_coders', '0069_listvalue_true_coders_coder_l_1cff61_idx'), + ('true_coders', '0070_coder_n_subscribers'), + ] + + operations = [ + migrations.RemoveIndex( + model_name='subscription', + name='notificatio_contest_559967_idx', + ), + migrations.RemoveField( + model_name='subscription', + name='account', + ), + migrations.AddField( + model_name='subscription', + name='accounts', + field=models.ManyToManyField(to='ranking.account'), + ), + migrations.AddIndex( + model_name='subscription', + index=models.Index(fields=['contest'], name='notificatio_contest_089a04_idx'), + ), + migrations.AddIndex( + model_name='subscription', + index=models.Index(fields=['resource'], name='notificatio_resourc_b72d99_idx'), + ), + migrations.AlterField( + model_name='notification', + name='coder', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='true_coders.coder'), + ), + migrations.AlterField( + model_name='subscription', + name='coder', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='true_coders.coder'), + ), + migrations.AddField( + model_name='subscription', + name='coders', + field=models.ManyToManyField(blank=True, related_name='subscribers', to='true_coders.coder'), + ), + migrations.AlterField( + model_name='subscription', + name='accounts', + field=models.ManyToManyField(blank=True, related_name='subscribers', to='ranking.account'), + ), + migrations.AddIndex( + model_name='subscription', + index=models.Index(fields=['enable', 'resource', 'contest'], name='notificatio_enable_2370eb_idx'), + ), + migrations.AddField( + model_name='subscription', + name='last_contest', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='last_contest_subscriptions', to='clist.contest'), + ), + migrations.AddField( + model_name='subscription', + name='last_update', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='subscription', + name='top_n', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='subscription', + name='with_first_accepted', + field=models.BooleanField(default=False), + ), + migrations.AddIndex( + model_name='subscription', + index=models.Index(fields=['resource', 'contest'], name='notificatio_resourc_b8cd18_idx'), + ), + migrations.AddIndex( + model_name='subscription', + index=models.Index(fields=['resource', 'contest', 'with_first_accepted'], name='notificatio_resourc_90d51f_idx'), + ), + migrations.AddIndex( + model_name='subscription', + index=models.Index(fields=['resource', 'contest', 'top_n'], name='notificatio_resourc_b039f3_idx'), + ), + ] diff --git a/src/notification/models.py b/src/notification/models.py index 581533b8..8fdb1b1b 100644 --- a/src/notification/models.py +++ b/src/notification/models.py @@ -11,20 +11,21 @@ from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models -from django.db.models.signals import pre_delete +from django.db.models.signals import m2m_changed, pre_delete from django.dispatch import receiver from django.template.loader import render_to_string from django.utils import timezone from clist.models import Contest, Resource -from pyclist.models import BaseModel +from pyclist.models import BaseManager, BaseModel from ranking.models import Account from tg.models import Chat as CoderChat from true_coders.models import Coder, CoderList +from utils.strings import markdown_to_html, markdown_to_text class TaskNotification(BaseModel): - coder = models.ForeignKey(Coder, on_delete=models.CASCADE) + coder = models.ForeignKey(Coder, on_delete=models.CASCADE, null=True, blank=True) method = models.CharField(max_length=256, null=False) enable = models.BooleanField(default=True) @@ -33,6 +34,23 @@ def method_type(self): ret, *_ = self.method.split(':', 1) return ret + def send(self, message=None, markdown=True, contest=None, **kwargs): + if markdown and message: + if self.method_type == django_settings.NOTIFICATION_CONF.WEBBROWSER: + message = markdown_to_text(message) + elif self.method_type == django_settings.NOTIFICATION_CONF.EMAIL: + message = markdown_to_html(message) + if contest is not None and self.last_contest != contest: + Task.create_contest_notification(notification=self, contest=contest) + self.last_contest = contest + self.last_update = timezone.now() + self.save(update_fields=['last_contest', 'last_update']) + Task.objects.create(notification=self, message=message, **kwargs) + + @property + def notification_key(self): + return f'{self.coder_id}:{self.method}' + class Meta: abstract = True @@ -78,7 +96,7 @@ class Notification(TaskNotification): ) def __str__(self): - return '{0.method}@{0.coder}: {0.before} {0.period}'.format(self) + return f'{self.method}@{self.coder}: {self.before} {self.period} Notification#{self.id}' def save(self, *args, **kwargs): if not self.id: @@ -98,12 +116,23 @@ def clean(self): raise ValidationError('WebBrowser method must have Event period.') +class EnabledSubscriptionManager(BaseManager): + def get_queryset(self): + return super().get_queryset().filter(enable=True) + + class Subscription(TaskNotification): resource = models.ForeignKey(Resource, null=True, blank=True, default=None, on_delete=models.CASCADE) contest = models.ForeignKey(Contest, null=True, blank=True, default=None, on_delete=models.CASCADE) - account = models.ForeignKey(Account, null=True, blank=True, default=None, on_delete=models.CASCADE) + coders = models.ManyToManyField(Coder, blank=True, related_name='subscribers') + accounts = models.ManyToManyField(Account, blank=True, related_name='subscribers') coder_list = models.ForeignKey(CoderList, null=True, blank=True, default=None, on_delete=models.CASCADE) coder_chat = models.ForeignKey(CoderChat, null=True, blank=True, default=None, on_delete=models.CASCADE) + last_contest = models.ForeignKey(Contest, null=True, blank=True, default=None, on_delete=models.CASCADE, + related_name='last_contest_subscriptions') + last_update = models.DateTimeField(null=True, blank=True) + with_first_accepted = models.BooleanField(default=False) + top_n = models.IntegerField(null=True, blank=True) tasks = GenericRelation( 'Task', @@ -112,8 +141,93 @@ class Subscription(TaskNotification): related_query_name='subscription', ) + objects = BaseManager() + enabled = EnabledSubscriptionManager() + class Meta: - indexes = [models.Index(fields=['contest', 'account'])] + indexes = [ + models.Index(fields=['contest']), + models.Index(fields=['resource']), + models.Index(fields=['enable', 'resource', 'contest']), + models.Index(fields=['resource', 'contest']), + models.Index(fields=['resource', 'contest', 'with_first_accepted']), + models.Index(fields=['resource', 'contest', 'top_n']), + ] + + def __str__(self): + ret = f'{self.method}@{self.coder}:' + if self.resource_id: + ret += f' {self.resource}' + if self.contest_id: + ret += f' {self.contest}' + return f'{ret} Subscription#{self.id}' + + def form_data(self): + ret = {} + ret['id'] = self.id + ret['method'] = {'id': self.method, 'text': self.method} + if self.resource: + ret['resource'] = {'id': self.resource.id, 'text': self.resource.host} + if self.contest: + ret['contest'] = {'id': self.contest.id, 'text': self.contest.title} + accounts = ret.setdefault('accounts', []) + for account in self.accounts.all(): + accounts.append({'id': account.id, 'text': account.display()}) + coders = ret.setdefault('coders', []) + for coder in self.coders.all(): + coders.append({'id': coder.id, 'text': coder.detailed_name}) + if self.coder_list_id: + ret['coder_list'] = {'id': self.coder_list.id, 'text': self.coder_list.name} + if self.coder_chat_id: + ret['coder_chat'] = {'id': self.coder_chat.id, 'text': self.coder_chat.title} + ret['with_first_accepted'] = self.with_first_accepted + ret['top_n'] = self.top_n + return ret + + def is_empty(self): + return not (self.accounts.exists() or self.coders.exists() or self.with_first_accepted or self.top_n) + + +@receiver(m2m_changed, sender=Subscription.accounts.through) +def update_account_n_subscribers_on_change(sender, instance, action, reverse, pk_set, **kwargs): + when, action = action.split('_', 1) + if when != 'post': + return + if action == 'add': + delta = 1 + elif action == 'remove': + delta = -1 + else: + return + + if reverse: + Account.objects.filter(pk=instance.pk).update(n_subscribers=models.F('n_subscribers') + delta) + elif pk_set: + Account.objects.filter(pk__in=pk_set).update(n_subscribers=models.F('n_subscribers') + delta) + + +@receiver(m2m_changed, sender=Subscription.coders.through) +def update_coder_n_subscribers_on_change(sender, instance, action, reverse, pk_set, **kwargs): + when, action = action.split('_', 1) + if when != 'post': + return + if action == 'add': + delta = 1 + elif action == 'remove': + delta = -1 + else: + return + + if reverse: + Coder.objects.filter(pk=instance.pk).update(n_subscribers=models.F('n_subscribers') + delta) + elif pk_set: + Coder.objects.filter(pk__in=pk_set).update(n_subscribers=models.F('n_subscribers') + delta) + + +@receiver(pre_delete, sender=Subscription) +def update_n_subscribers_on_delete(sender, instance, **kwargs): + Account.objects.filter(subscribers=instance).update(n_subscribers=models.F('n_subscribers') - 1) + Coder.objects.filter(subscribers=instance).update(n_subscribers=models.F('n_subscribers') - 1) class Task(BaseModel): @@ -138,8 +252,14 @@ def get_queryset(self): objects = ObjectsManager() unsent = UnsentManager() + @classmethod + def create_contest_notification(cls, notification, contest, **kwargs): + addition = kwargs.setdefault('addition', {}) + addition['contests'] = [contest.pk] + Task.objects.create(notification=notification, **kwargs) + def __str__(self): - return 'task of {0.notification}'.format(self) + return 'Task#{0.id} {0.notification}'.format(self) @receiver(pre_delete, sender=Task) diff --git a/src/notification/utils.py b/src/notification/utils.py new file mode 100644 index 00000000..ae17687d --- /dev/null +++ b/src/notification/utils.py @@ -0,0 +1,100 @@ +import re + +import django_rq +import flag +from django.conf import settings +from django.contrib.auth.models import User +from django.core.management import call_command +from django.urls import reverse + +from clist.templatetags.extras import (as_number, get_division_problems, get_problem_name, get_problem_short, is_hidden, + is_partial, is_solved, md_escape, md_url, md_url_text, scoreformat, + solution_time_compare) + + +def compose_message_by_problems(problem_shorts, statistic, previous_addition, contest_or_problems): + problems = statistic.addition.get('problems', {}) + previous_problems = previous_addition.get('problems', {}) + + contest_problems = get_division_problems(contest_or_problems, statistic.addition) + contest_problems = {get_problem_short(problem): problem for problem in contest_problems} + + problem_messages = [] + max_time_solution = None + is_improving = False + + if problem_shorts == 'all': + problem_shorts = list(problems.keys()) + else: + for short, solution in problems.items(): + if short in problem_shorts or not is_hidden(solution): + continue + problem_shorts.append(short) + + for short in problem_shorts: + solution = problems.get(short, {}) + previous_problem = previous_problems.get(short, {}) + + result = solution.get('result') + verdict = solution.get('verdict') + previous_result = previous_problem.get('result') + + is_solved_result = is_solved(solution) + is_improving |= is_solved_result + + contest_problem = contest_problems.get(short, {}) + problem_message = short if 'short' in contest_problem else get_problem_name(contest_problem) + problem_message = md_escape(problem_message) + if re.match(r'^[\w\d_]+$', problem_message): + problem_message = f'#{problem_message}' + if 'name' in solution: + problem_message = '%s. %s' % (problem_message, md_escape(solution['name'])) + problem_message = '%s `%s`' % (problem_message, scoreformat(result, with_shorten=False)) + if verdict: + problem_message = '%s %s' % (problem_message, md_escape(verdict)) + + if previous_result and is_partial(solution): + delta = as_number(result, default=0) - as_number(previous_result, default=0) + is_improving |= delta > 0 + problem_message += " `%s%s`" % ('+' if delta >= 0 else '', delta) + + if max_time_solution is None or solution_time_compare(max_time_solution, solution) < 0: + max_time_solution = solution + + if solution.get('first_ac'): + problem_message += ' FIRST ACCEPTED' + if solution.get('is_max_score'): + problem_message += ' MAX SCORE' + if solution.get('try_first_ac'): + problem_message += ' TRY FIRST AC' + problem_messages.append(problem_message) + problem_message = '(%s)' % ', '.join(problem_messages) if problem_messages else '' + time_message = '`[%s]`' % max_time_solution["time"] if max_time_solution and 'time' in max_time_solution else '' + + previous_place = previous_addition.get('place') + previous_solving = previous_addition.get('score') + has_solving_diff = previous_solving is None or previous_solving != statistic.solving + place_message = statistic.place + if previous_place and has_solving_diff: + place_message = '%s->%s' % (previous_place, statistic.place) + + standings_url = reverse('ranking:standings_by_id', args=[statistic.contest_id]) + f'?find_me={statistic.pk}' + account_message = '[%s](%s)' % (md_url_text(statistic.account_name), md_url(standings_url)) + if statistic.account.country: + account_message = flag.flag(statistic.account.country.code) + account_message + + suffix_message = '' + if has_solving_diff: + suffix_message += f'= `{scoreformat(statistic.solving, with_shorten=False)}`' + if 'penalty' in statistic.addition: + suffix_message += rf' `[{statistic.addition["penalty"]}]`' + message = f'{time_message} `{place_message}`. {account_message} {suffix_message} {problem_message}'.strip() + message = re.sub(r'(\s)\s+', r'\1', message) + return message + + +def send_messages(**kwargs): + if settings.DEBUG: + admin_coders = list(User.objects.filter(is_superuser=True).values_list('username', flat=True)) + kwargs.setdefault('coders', []).extend(admin_coders) + django_rq.get_queue('system').enqueue(call_command, 'sendout_tasks', **kwargs) diff --git a/src/pyclist/decorators.py b/src/pyclist/decorators.py index 12a2679d..f84d91c5 100644 --- a/src/pyclist/decorators.py +++ b/src/pyclist/decorators.py @@ -14,42 +14,76 @@ from stringcolor import bold, cs from clist.models import Contest -from clist.templatetags.extras import query_transform, slug, toint +from clist.templatetags.extras import slug, toint, trim_to +from utils.strings import slug_string_iou logger = logging.getLogger(__name__) +def get_contest_by_id_and_slug(contests, contest_id, title_slug) -> None | Contest: + contest = contests.filter(pk=contest_id).first() + if contest is None: + return + if slug_string_iou(title_slug, slug(contest.title)) < 0.3: + return + return contest + + +def get_contest_by_series(contests, series) -> None | Contest: + contest = contests.filter(series__aliases__contains=slug(series)).order_by('-end_time').first() + if contest is None: + return + return contest + + +def guess_contest(contests, contest_id, title_slug): + contests_iterator = contests.filter(slug=title_slug).iterator() + + try: + contest = None + contest = next(contests_iterator) + another = next(contests_iterator) + except StopIteration: + another = None + + if another is not None: + return redirect(reverse('ranking:standings_list') + f'?search=slug:{title_slug}') + + if contest is not None: + return contest + + contest = get_contest_by_id_and_slug(contests, contest_id, title_slug) + if contest is not None: + return contest + + contest = get_contest_by_series(contests, title_slug) + if contest is not None: + return contest + + return HttpResponseNotFound() + + def inject_contest(): def decorator(view): @wraps(view) def decorated(request, title_slug=None, contest_id=None, contests_ids=None, *args, **kwargs): contests = Contest.objects.annotate_favorite(request.user) - to_redirect = False + to_canonical = False contest = None if contest_id is not None: contest = contests.filter(pk=contest_id).first() if title_slug is None: - to_redirect = True + to_canonical = True else: if contest is None or slug(contest.title) != title_slug: contest = None title_slug += f'-{contest_id}' if contest is None and title_slug is not None: - contests_iterator = contests.filter(slug=title_slug).iterator() - - contest = None - try: - contest = next(contests_iterator) - another = next(contests_iterator) - except StopIteration: - another = None - if contest is None: - return HttpResponseNotFound() - if another is None: - to_redirect = True - else: - return redirect(reverse('ranking:standings_list') + f'?search=slug:{title_slug}') + contest = guess_contest(contests, contest_id, title_slug) + if not isinstance(contest, Contest): + return contest + to_canonical = True if contests_ids is not None: cids, contests_ids = list(map(toint, contests_ids.split(','))), [] @@ -58,18 +92,16 @@ def decorated(request, title_slug=None, contest_id=None, contests_ids=None, *arg contests_ids.append(cid) contest = contests.filter(pk=contests_ids[0]).first() kwargs['other_contests'] = list(contests.filter(pk__in=contests_ids[1:])) + if contest is None: return HttpResponseNotFound() - if to_redirect: + if to_canonical: resolved = resolve(request.path) viewname = resolved.app_name + ':' + resolved.url_name.split('_')[0] - query = query_transform(request) url = reverse(viewname, kwargs={'title_slug': slug(contest.title), 'contest_id': str(contest.pk)}) - if query: - query = '?' + query - return redirect(url + query) + request.set_canonical(url) response = view(request, *args, contest=contest, **kwargs) return response @@ -136,7 +168,7 @@ def log_grouped_times(grouped_times): msg = bold(f'{g["sum"]:.3f}') + ' ms' msg += ' (avg ' + bold(f'{g["avg"]:.3f}') + ' ms)' msg += ' ' + bold(f'{g["count"]}') + ' times' - msg += ': ' + cs(g['query'], 'grey') + msg += ': ' + cs(trim_to(g['query'], 100, raw_text=True), 'grey') print(msg) diff --git a/src/pyclist/middleware.py b/src/pyclist/middleware.py index c0fd3ce3..8717f2f6 100644 --- a/src/pyclist/middleware.py +++ b/src/pyclist/middleware.py @@ -6,15 +6,14 @@ from django.conf import settings from django.db import connection from django.db.models import Q -from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect +from django.http import HttpResponse, HttpResponseForbidden from django.middleware import csrf -from django.urls import reverse from django.utils import timezone from pytz import timezone as pytz_timezone from clist.templatetags.extras import redirect_login from true_coders.models import Coder -from utils.custom_request import custom_request +from utils.custom_request import CustomRequest def DebugPermissionOnlyMiddleware(get_response): @@ -23,7 +22,7 @@ def middleware(request): first_path = request.path.split('/')[1] if first_path not in settings.DEBUG_PERMISSION_EXCLUDE_PATHS: if not request.user.is_authenticated: - if first_path not in ('login', 'signup', 'oauth', 'calendar', 'telegram'): + if first_path not in ('login', 'signup', 'oauth', 'calendar', 'telegram', 'form'): return redirect_login(request) elif not request.user.has_perm('auth.view_debug'): return HttpResponseForbidden() @@ -43,7 +42,7 @@ def middleware(request): def CustomRequestMiddleware(get_response): def middleware(request): - request = custom_request(request) + request = CustomRequest(request) response = get_response(request) return response diff --git a/src/pyclist/models.py b/src/pyclist/models.py index 37c4e463..7a6929c8 100644 --- a/src/pyclist/models.py +++ b/src/pyclist/models.py @@ -1,3 +1,5 @@ +from typing import Any, Optional + from django.apps import apps from django.contrib.auth.models import AnonymousUser, User from django.contrib.contenttypes.models import ContentType @@ -25,6 +27,15 @@ class BaseModel(models.Model): created = models.DateTimeField(auto_now_add=True, db_index=True) modified = models.DateTimeField(auto_now=True, db_index=True) + def fetched_field(self, field) -> Optional[Any]: + fields = field.split('__') + obj = self + for field in fields: + if field not in obj._state.fields_cache: + return + obj = getattr(obj, field) + return obj + class Meta: abstract = True @@ -67,7 +78,6 @@ def annotate_note(self, instance): content_type = ContentType.objects.get_for_model(self.model) qs = Note.objects.filter(coder=coder, content_type=content_type, object_id=OuterRef('pk')) ret = self.annotate(is_note=Exists(qs)) - ret = ret.annotate(note_text=Subquery(qs.values('text')[:1])) return ret diff --git a/src/pyclist/settings.py b/src/pyclist/settings.py index 60a11001..073ac19e 100644 --- a/src/pyclist/settings.py +++ b/src/pyclist/settings.py @@ -180,6 +180,7 @@ 'favorites.templatetags.favorites_extras', 'django.templatetags.cache', 'el_pagination.templatetags.el_pagination_tags', + 'jsonify.templatetags.jsonify', ], 'loaders': [ ('django.template.loaders.cached.Loader', [ @@ -212,11 +213,16 @@ # django_rq RQ_QUEUES = { + 'system': { + 'HOST': 'localhost', + 'PORT': 6379, + 'DEFAULT_TIMEOUT': 60, + }, 'default': { 'HOST': 'localhost', 'PORT': 6379, - 'DEFAULT_TIMEOUT': 3600, - } + 'DEFAULT_TIMEOUT': 300, + }, } RQ_SHOW_ADMIN_LINK = True @@ -282,7 +288,7 @@ RESOURCES_ICONS_SIZES = [32, 64] STORAGES = { - "default": {"BACKEND": "django.core.files.storage.FileystemStorage"}, + 'default': {'BACKEND': 'django.core.files.storage.FileSystemStorage'}, 'staticfiles': {'BACKEND': 'static_compress.CompressedStaticFilesStorage'}, } @@ -592,6 +598,9 @@ def show_toolbar_callback(request): CSP_IMG_SRC += ('https://www.google-analytics.com', ) CSP_CONNECT_SRC += ('https://www.google-analytics.com', ) +# CSP Yandex form +CSP_SCRIPT_SRC += ('https://forms.yandex.ru', ) + # X-XSS-Protection SECURE_BROWSER_XSS_FILTER = True @@ -657,6 +666,9 @@ def show_toolbar_callback(request): DEFAULT_API_THROTTLE_AT_ = 10 CODER_LIST_N_VALUES_LIMIT_ = 100 +CODER_N_SUBSCRIPTIONS_LIMIT_ = 10 +CODER_SUBSCRIPTION_N_LIMIT_ = CODER_LIST_N_VALUES_LIMIT_ +CODER_SUBSCRIPTION_TOP_N_LIMIT_ = 50 ENABLE_GLOBAL_RATING_ = False @@ -780,6 +792,13 @@ def show_toolbar_callback(request): 'loading': '<i class="fas fa-circle-notch fa-spin"></i>', 'profile': dict(icon='<i class="fa-regular fa-address-card"></i>', title=False), 'rating_prediction': '<i class="fa-solid fa-calculator"></i>', + 'https': '<i class="fa-regular fa-square-check"></i>', + 'http': '<i class="fa-regular fa-rectangle-xmark"></i>', + 'edit': '<i class="fa-regular fa-pen-to-square"></i>', + 'login': '<i class="fa-solid fa-right-to-bracket"></i>', + 'logout': '<i class="fa-solid fa-right-from-bracket"></i>', + 'expires': '<i class="fa-solid fa-clock-rotate-left"></i>', + 'subscription': '<i class="fa-regular fa-newspaper"></i>', 'google': {'icon': '<i class="fab fa-google"></i>', 'title': None}, 'facebook': {'icon': '<i class="fab fa-facebook"></i>', 'title': None}, @@ -790,7 +809,7 @@ def show_toolbar_callback(request): 'discord': {'icon': '<i class="fab fa-discord"></i>', 'title': None}, 'vk': {'icon': '<i class="fab fa-vk"></i>', 'title': None}, 'patreon': {'icon': '<i class="fab fa-patreon"></i>', 'title': None}, - 'competitive-hustle': {'icon': '<i class="fas fa-tools"></i>'}, + 'yandex-contest': {'icon': '<i class="fas fa-tools"></i>'}, } @@ -822,9 +841,9 @@ class NOTIFICATION_CONF: WEBBROWSER = 'webbrowser' METHODS_CHOICES = ( - (EMAIL, 'Email'), (TELEGRAM, 'Telegram'), (WEBBROWSER, 'WebBrowser'), + # (EMAIL, 'Email'), ) @@ -843,8 +862,8 @@ class NOTIFICATION_CONF: DjangoIntegration(), LoggingIntegration(level=logging.INFO, event_level=logging.ERROR), ], - traces_sample_rate=0.01, - profiles_sample_rate=0.01, + traces_sample_rate=0.005, + profiles_sample_rate=0.005, send_default_pii=True, environment='development' if DEBUG else 'production', ) diff --git a/src/ranking/admin.py b/src/ranking/admin.py index 5a2777e3..319d12f7 100644 --- a/src/ranking/admin.py +++ b/src/ranking/admin.py @@ -6,7 +6,8 @@ from pyclist.admin import BaseModelAdmin, admin_register from ranking.management.commands.parse_statistic import Command as parse_stat from ranking.models import (Account, AccountMatching, AccountRenaming, AccountVerification, AutoRating, CountryAccount, - Module, Rating, Stage, StageContest, Statistics, VerifiedAccount, VirtualStart) + Module, ParseStatistics, Rating, Stage, StageContest, Statistics, VerifiedAccount, + VirtualStart) class HasCoders(admin.SimpleListFilter): @@ -59,8 +60,9 @@ def _has_coder(self, obj): def get_readonly_fields(self, request, obj=None): return ( - ['updated', 'n_contests', 'n_writers', 'last_activity', 'last_submission', 'last_rating_activity'] + - super().get_readonly_fields(request, obj) + ['updated', 'n_contests', 'n_writers', 'n_subscribers', + 'last_activity', 'last_submission', 'last_rating_activity'] + + super().get_readonly_fields(request, obj) ) def get_queryset(self, request): @@ -144,13 +146,6 @@ class StageAdmin(BaseModelAdmin): list_filter = ['contest__host'] ordering = ['-contest__start_time'] - def parse_stage(self, request, queryset): - for stage in queryset: - stage.update() - parse_stage.short_description = 'Parse stages' - - actions = [parse_stage] - class StageContestInline(admin.TabularInline): model = StageContest raw_id_fields = ['contest'] @@ -221,3 +216,13 @@ def _n_different_coders(self, obj): return obj.n_different_coders _n_different_coders.admin_order_field = 'n_different_coders' _n_different_coders.short_description = 'NDC' + + +@admin_register(ParseStatistics) +class ParseStatisticsAdmin(BaseModelAdmin): + list_display = ['contest', 'enable', 'delay', 'created', 'modified'] + search_fields = ['contest__title', 'contest__host'] + list_filter = ['contest__host'] + + def get_readonly_fields(self, *args, **kwargs): + return ['parse_time'] + super().get_readonly_fields(*args, **kwargs) diff --git a/src/ranking/management/commands/informer.py b/src/ranking/management/commands/informer.py index 213a6347..908026e5 100644 --- a/src/ranking/management/commands/informer.py +++ b/src/ranking/management/commands/informer.py @@ -8,7 +8,6 @@ import subprocess import time from datetime import timedelta -from numbers import Number from pprint import pprint # noqa import coloredlogs @@ -17,6 +16,7 @@ from django.conf import settings from django.core.management import call_command from django.core.management.base import BaseCommand +from django.db.models import Q from django.urls import reverse from django.utils import timezone from rich.console import Console @@ -24,8 +24,9 @@ from rich.table import Table from clist.models import Contest -from clist.templatetags.extras import (as_number, get_problem_name, get_problem_short, has_season, md_escape, - md_italic_escape, md_url, scoreformat) +from clist.templatetags.extras import (as_number, get_problem_name, get_problem_short, has_season, md_escape, md_url, + scoreformat, solution_time_compare) +from notification.models import Subscription from ranking.models import Statistics from tg.bot import MAX_MESSAGE_LENGTH, Bot, telegram from tg.models import Chat @@ -76,6 +77,8 @@ def add_arguments(self, parser): parser.add_argument('--dryrun', action='store_true', help='Do not send to bot message', default=False) parser.add_argument('--dump', help='Dump and restore log file', default=None) parser.add_argument('--force-iterations', type=int, help='Minimum number of iterations before stop', default=0) + parser.add_argument('--with-history', action='store_true', help='Show history', default=False) + parser.add_argument('--with-subscriptions', action='store_true', help='Send to subscriptions', default=False) parser.add_argument('--no-parse-statistic', action='store_true', help='Do not call command to parse statistics', default=False) @@ -110,7 +113,7 @@ def handle(self, *args, **options): print(now) if not args.no_parse_statistic: - call_command('parse_statistic', contest_id=args.cid, without_fill_coder_problems=True) + call_command('parse_statistic', contest_id=args.cid, without_set_coder_problems=True) contest = Contest.objects.get(pk=args.cid) resource = contest.resource qs = Statistics.objects.filter(contest=contest) @@ -204,25 +207,6 @@ def delete_message(): in_time = None - def time_compare(lhs, rhs): - - def get_value(val): - if isinstance(val, Number): - val = [val] - else: - val = list(val.split(':')) - return len(val), val - - for k in ('time_in_seconds', 'time'): - if k not in lhs or k not in rhs: - continue - l_val = get_value(lhs[k]) - r_val = get_value(rhs[k]) - if l_val != r_val: - return -1 if l_val < r_val else 1 - - return 0 - for k, v in stat_problems.items(): if 'result' not in v: continue @@ -261,14 +245,14 @@ def get_value(val): m = '%s%s `%s`' % (short, ('. ' + v['name']) if 'name' in v else '', scoreformat(result)) if verdict: - m += ' ' + md_italic_escape(verdict) + m += ' ' + md_escape(verdict) if has_change: if p_result is not None and v.get('partial'): delta = as_number(result) - (as_number(p_result) or 0) m += " `%s%s`" % ('+' if delta >= 0 else '', delta) - if in_time is None or time_compare(in_time, v) < 0: + if in_time is None or solution_time_compare(in_time, v) < 0: in_time = v if verdict not in ['U']: @@ -343,8 +327,9 @@ def get_value(val): if filtered: table.add_row(str(stat.place), str(stat.solving), Markdown(msg), str(stat.pk)) if not args.dryrun and (filtered and has_update or has_first_ac or has_try_first_ac or has_max_score): - delete_message() - msg = combine_messages(msg, message_text) + if args.with_history: + delete_message() + msg = combine_messages(msg, message_text) n_attempts = 5 delay_on_timeout = 3 for it in range(n_attempts): @@ -357,10 +342,21 @@ def get_value(val): except telegram.error.BadRequest as e: logger.error(str(e)) wait('send message', it * delay_on_timeout) - continue if not skip_in_message_text: message_text = combine_messages(history_msg, message_text) + if args.with_subscriptions: + has_subscription_update = stat.account.n_subscribers and has_update + if has_subscription_update: + subscriptions_filter = ( + Q(accounts=stat.account) + & (Q(resource__isnull=True) | Q(resource=resource)) + & (Q(contest__isnull=True) | Q(contest=contest)) + ) + subscriptions = Subscription.objects.filter(subscriptions_filter) + for subscription in subscriptions: + subscription.send(message=msg) + data = { 'solving': stat.solving, 'place': stat.place, diff --git a/src/ranking/management/commands/parse_accounts_infos.py b/src/ranking/management/commands/parse_accounts_infos.py index de3e6018..feca06f6 100644 --- a/src/ranking/management/commands/parse_accounts_infos.py +++ b/src/ranking/management/commands/parse_accounts_infos.py @@ -258,7 +258,7 @@ def inf_none(): coders = list(account.coders.values_list('username', flat=True)) if coders and updated_info.get('n_updated'): - call_command('fill_coder_problems', coders=coders, resources=[resource.host]) + call_command('set_coder_problems', coders=coders, resources=[resource.host]) if info.get('country'): account.country = countrier.get(info['country']) @@ -310,7 +310,7 @@ def inf_none(): try: updated = arrow.get(now + timedelta(days=1)).ceil('day').datetime for a in tqdm(accounts, desc='changing update time'): - if a.key not in seen: + if a.pk and a.key not in seen: a.updated = updated a.save(update_fields=['updated']) n_counter['reupdated'] += 1 diff --git a/src/ranking/management/commands/parse_live_statistics.py b/src/ranking/management/commands/parse_live_statistics.py new file mode 100644 index 00000000..bd57384d --- /dev/null +++ b/src/ranking/management/commands/parse_live_statistics.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import logging +# import os +# import re +# import subprocess +import time + +# import coloredlogs +# import flag +import humanize +# from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand +# from django.db.models import Q +# from django.urls import reverse +from django.utils import timezone + +from ranking.models import ParseStatistics +# from clist.models import Contest +# from clist.templatetags.extras import (as_number, get_problem_name, get_problem_short, has_season, md_escape, md_url, +# scoreformat, solution_time_compare) +# from notification.models import Subscription +# from ranking.models import Statistics +# from tg.bot import MAX_MESSAGE_LENGTH, Bot, telegram +# from tg.models import Chat +# from true_coders.models import CoderList +from utils.attrdict import AttrDict + +# from datetime import timedelta +# from pprint import pprint # noqa + +# from rich.console import Console +# from rich.markdown import Markdown +# from rich.table import Table + +# from utils.strings import trim_on_newline + + +class Command(BaseCommand): + help = 'Parse live statistics' + + def add_arguments(self, parser): + parser.add_argument('--dryrun', action='store_true', default=False) + self.logger = logging.getLogger('ranking.parse.live_statistics') + + def handle(self, *args, **options): + self.stdout.write(str(options)) + args = AttrDict(options) + logging.disable(logging.DEBUG) + + while True: + now = timezone.now() + parse_statistics = ParseStatistics.objects.filter( + enable=True, + contest__start_time__lte=now, + contest__end_time__gte=now, + ) + parse_statistics = parse_statistics.select_related('contest') + + for parse_stat in parse_statistics: + now = timezone.now() + if parse_stat.parse_time and parse_stat.parse_time > now: + self.logger.info(f'Skip {parse_stat.contest}, delay = {parse_stat.parse_time - now}') + continue + self.logger.info(f'Parse statistic for {parse_stat.contest}') + parse_stat.parse_time = now + parse_stat.delay + parse_stat.save(update_fields=['parse_time']) + if args.dryrun: + continue + call_command( + 'parse_statistic', + contest_id=parse_stat.contest.pk, + without_set_coder_problems=parse_stat.without_set_coder_problems, + without_stage=parse_stat.without_stage, + without_subscriptions=parse_stat.without_subscriptions, + ) + + parse_times = [p.parse_time for p in parse_statistics] + if not parse_times: + break + + parse_time = min(parse_times) + delay = (parse_time - now).total_seconds() + self.logger.info(f'Delay = {humanize.naturaldelta(delay)}, next parse time = {parse_time}') + time.sleep(delay) + + # if not re.match(r'^-?\d+$', args.tid): + # tg_chat_id = Chat.objects.get(title=args.tid).chat_id + # else: + # tg_chat_id = args.tid + + # bot = Bot() + + # if args.dump is not None and os.path.exists(args.dump): + # with open(args.dump, 'r') as fo: + # standings = json.load(fo) + # else: + # standings = {} + + # problems_info = standings.setdefault('__problems_info', {}) + + # iteration = 1 if args.dump else 0 + # forced_iterations = args.force_iterations + + # console = Console() + + # while True: + # subprocess.call('clear', shell=True) + # now = timezone.now() + # print(now) + + # if not args.no_parse_statistic: + # call_command('parse_statistic', contest_id=args.cid, without_set_coder_problems=True) + # contest = Contest.objects.get(pk=args.cid) + # resource = contest.resource + # qs = Statistics.objects.filter(contest=contest) + # qs = qs.prefetch_related('account') + # statistics = list(qs) + + # for p in problems_info.values(): + # if p.get('accepted') or not p.get('n_hidden'): + # p.pop('show_hidden', None) + # p['n_hidden'] = 0 + + # updated = False + # has_hidden = contest.has_hidden_results + # numbered = 0 + # table = Table() + # table.add_column('Rank') + # table.add_column('Score') + # table.add_column('Text') + # table.add_column('Pk') + + # def statistics_sort_key(stat): + # return ( + # stat.place_as_int if stat.place_as_int is not None else float('inf'), + # -stat.solving, + # ) + + # if args.coder_list: + # accounts_filter = CoderList.accounts_filter(uuids=[args.coder_list]) + # qs = resource.account_set.filter(accounts_filter) # TODO filter by contest + # coder_list_accounts = set(qs.values_list('pk', flat=True)) + + # for stat in sorted(statistics, key=statistics_sort_key): + # name_instead_key = resource.info.get('standings', {}).get('name_instead_key') + # name_instead_key = stat.account.info.get('_name_instead_key', name_instead_key) + + # if name_instead_key: + # name = stat.account.name + # else: + # name = stat.addition.get('name') + # if not name or not has_season(stat.account.key, name): + # name = stat.account.key + + # stat_problems = stat.addition.get('problems', {}) + + # filtered = False + # if args.query is not None and re.search(args.query, name, re.I): + # filtered = True + # if args.top and stat.place_as_int and stat.place_as_int <= args.top and stat_problems: + # filtered = True + # if args.coder_list and stat.account.pk in coder_list_accounts: + # filtered = True + + # contest_problems = contest.info.get('problems') + # division = stat.addition.get('division') + # if division and 'division' in contest_problems: + # contest_problems = contest_problems['division'][division] + # contest_problems = {get_problem_short(p): p for p in contest_problems} + + # key = str(stat.account.id) + # standings_key = standings.get(key, {}) + # problems = standings_key.get('problems', {}) + # message_id = standings_key.get('message_id') + # message_text = standings_key.get('message_text', '') + # skip_in_message_text = True + + # def delete_message(): + # nonlocal message_id + # if message_id: + # for it in range(3): + # try: + # bot.delete_message(chat_id=tg_chat_id, message_id=message_id) + # message_id = None + # break + # except telegram.error.BadRequest as e: + # logger.warning(str(e)) + # if 'Message to delete not found' in str(e): + # break + # raise e + # except telegram.error.TimedOut as e: + # logger.warning(str(e)) + # time.sleep(it) + # continue + + # p = [] + # has_update = False + # has_first_ac = False + # has_max_score = False + # has_try_first_ac = False + # has_top = False + # has_solving_diff = ( + # 'solving' not in standings_key or + # abs(standings_key['solving'] - stat.solving) > 1e-9 + # ) + + # in_time = None + + # for k, v in stat_problems.items(): + # if 'result' not in v: + # continue + + # p_info = problems_info.setdefault(k, {}) + # p_result = problems.get(k, {}).get('result') + # p_verdict = problems.get(k, {}).get('verdict') + # result = v['result'] + # verdict = v.get('verdict') + + # is_hidden = str(result).startswith('?') + # is_accepted = str(result).startswith('+') or v.get('binary', False) + # try: + # is_accepted = is_accepted or float(result) > 0 and not v.get('partial') + # except Exception: + # pass + # is_max_score = False + # try: + # is_max_score = float(result) > p_info.get('max_score', 0) + # except Exception: + # pass + + # if is_hidden: + # p_info['n_hidden'] = p_info.get('n_hidden', 0) + 1 + + # has_change = p_result != result or (p_verdict and verdict and p_verdict != verdict) + # if has_change or is_hidden: + # short = k + # contest_problem = contest_problems.get(k, {}) + # if contest_problem and ('short' not in contest_problem or short != get_problem_short(contest_problem)): # noqa + # short = get_problem_name(contest_problems[k]) + # short = md_escape(short) + # if 'url' in contest_problem: + # short = '[%s](%s)' % (short, contest_problem['url']) + + # m = '%s%s `%s`' % (short, ('. ' + v['name']) if 'name' in v else '', scoreformat(result)) + + # if verdict: + # m += ' ' + md_escape(verdict) + + # if has_change: + # if p_result is not None and v.get('partial'): + # delta = as_number(result) - (as_number(p_result) or 0) + # m += " `%s%s`" % ('+' if delta >= 0 else '', delta) + + # if in_time is None or solution_time_compare(in_time, v) < 0: + # in_time = v + + # if verdict not in ['U']: + # skip_in_message_text = False + + # has_update = True + # if iteration: + # if p_info.get('show_hidden') == key: + # delete_message() + # if not is_hidden: + # p_info.pop('show_hidden') + # if not p_info.get('accepted'): + # if is_accepted: + # m += ' FIRST ACCEPTED' + # has_first_ac = True + # elif is_max_score: + # m += ' MAX SCORE' + # has_max_score = True + # elif is_hidden and not p_info.get('show_hidden'): + # p_info['show_hidden'] = key + # m += ' TRY FIRST AC' + # has_try_first_ac = True + # if args.top and stat.place_as_int and stat.place_as_int <= args.top: + # has_top = True + # p.append(m) + # if is_accepted: + # p_info['accepted'] = True + # if is_max_score: + # p_info['max_score'] = float(result) + # has_hidden = has_hidden or is_hidden + + # prev_place_as_int = standings_key.get('place_as_int') + # place_as_int = stat.place_as_int + # if prev_place_as_int and place_as_int and prev_place_as_int > place_as_int: + # has_update = True + # prev_place = standings_key.get('place') + # place = stat.place + # if has_solving_diff and prev_place: + # place = '%s->%s' % (prev_place, place) + # if args.numbered is not None and re.search(args.numbered, stat.account.key, re.I): + # numbered += 1 + # place = '%s (%s)' % (place, numbered) + + # prefix_msg = '' + # if place is not None: + # prefix_msg += '`%s`. ' % place + # if in_time and 'time' in in_time: + # prefix_msg = '`[%s]` %s' % (in_time['time'], prefix_msg) + + # account_url = reverse('coder:account', kwargs={'key': stat.account.key, 'host': resource.host}) + # account_url = settings.MAIN_HOST_URL_ + md_url(account_url) + # account_msg = '[%s](%s)' % (name, account_url) + # if stat.account.country: + # account_msg = flag.flag(stat.account.country.code) + account_msg + + # suffix_msg = '' + # if has_solving_diff: + # suffix_msg += ' = `%d`' % stat.solving + # if 'penalty' in stat.addition: + # suffix_msg += rf' `[{stat.addition["penalty"]}]`' + # if p: + # suffix_msg += ' (%s)' % ', '.join(p) + # if has_top: + # suffix_msg += f' TOP{args.top}' + + # if has_update or has_first_ac or has_try_first_ac or has_max_score: + # updated = True + + # msg = prefix_msg + account_msg + suffix_msg + # history_msg = prefix_msg.rstrip() + suffix_msg + + # if filtered: + # table.add_row(str(stat.place), str(stat.solving), Markdown(msg), str(stat.pk)) + # if not args.dryrun and (filtered and has_update or has_first_ac or has_try_first_ac or has_max_score): + # if args.with_history: + # delete_message() + # msg = combine_messages(msg, message_text) + # n_attempts = 5 + # delay_on_timeout = 3 + # for it in range(n_attempts): + # try: + # message = bot.send_message(msg=msg, chat_id=tg_chat_id) + # message_id = message.message_id + # break + # except telegram.error.TimedOut as e: + # logger.warning(str(e)) + # except telegram.error.BadRequest as e: + # logger.error(str(e)) + # wait('send message', it * delay_on_timeout) + # if not skip_in_message_text: + # message_text = combine_messages(history_msg, message_text) + + # if args.with_subscriptions: + # has_subscription_update = stat.account.n_subscribers and has_update + # if has_subscription_update: + # subscriptions_filter = ( + # Q(accounts=stat.account) + # & (Q(resource__isnull=True) | Q(resource=resource)) + # & (Q(contest__isnull=True) | Q(contest=contest)) + # ) + # subscriptions = Subscription.objects.filter(subscriptions_filter) + # for subscription in subscriptions: + # subscription.send(message=msg) + + # data = { + # 'solving': stat.solving, + # 'place': stat.place, + # 'place_as_int': stat.place_as_int, + # 'problems': stat.addition.get('problems', {}), + # 'message_id': message_id, + # 'message_text': message_text, + # } + # standings[key] = data + + # console.print(table) + + # if args.dump is not None and (updated or not os.path.exists(args.dump)): + # standings_dump = json.dumps(standings, indent=2) + # with open(args.dump, 'w') as fo: + # fo.write(standings_dump) + + # if iteration: + # is_over = contest.end_time < now or contest.time_percentage >= 1 + # if args.no_parse_statistic or forced_iterations <= 0 and is_over and not has_hidden: + # break + # is_coming = now < contest.standings_start_time + # if is_coming: + # limit = contest.standings_start_time + # else: + # tick = args.delay * 10 if is_over else args.delay + # limit = now + timedelta(seconds=tick) + # wait('parsing iteration', limit) + + # iteration += 1 + # forced_iterations -= 1 diff --git a/src/ranking/management/commands/parse_statistic.py b/src/ranking/management/commands/parse_statistic.py index 122fc58a..e9b04a53 100644 --- a/src/ranking/management/commands/parse_statistic.py +++ b/src/ranking/management/commands/parse_statistic.py @@ -22,7 +22,7 @@ from django.core.management import call_command from django.core.management.base import BaseCommand from django.db import transaction -from django.db.models import Exists, F, OuterRef, Q +from django.db.models import Exists, F, Max, OuterRef, Prefetch, Q from django.utils.timezone import now as timezone_now from django_print_sql import print_sql_decorator @@ -30,17 +30,18 @@ from clist.models import Contest, Problem, Resource from clist.templatetags.extras import (as_number, canonize, get_item, get_number_from_str, get_problem_key, - get_problem_short, is_solved, normalize_field, time_in_seconds, + get_problem_short, is_hidden, is_solved, normalize_field, time_in_seconds, time_in_seconds_format) from clist.views import update_problems, update_writers from logify.models import EventLog, EventStatus from notification.models import NotificationMessage, Subscription +from notification.utils import compose_message_by_problems, send_messages from pyclist.decorators import analyze_db_queries from ranking.management.commands.parse_accounts_infos import rename_account from ranking.management.modules.common import REQ, UNCHANGED, ProxyLimitReached from ranking.management.modules.excepts import ExceptionParseStandings, InitModuleException from ranking.models import Account, AccountRenaming, Module, Stage, Statistics -from ranking.utils import account_update_contest_additions +from ranking.utils import account_update_contest_additions, update_stage from ranking.views import update_standings_socket from true_coders.models import Coder from utils.attrdict import AttrDict @@ -197,10 +198,11 @@ def add_arguments(self, parser): parser.add_argument('--before-date', default=False, help='Update contests that have been updated to date') parser.add_argument('--after-date', default=False, help='Update contests that have been updated after date') parser.add_argument('--with-medals', action='store_true', default=False, help='Contest with medals') - parser.add_argument('--without-fill-coder-problems', action='store_true', default=False) + parser.add_argument('--without-set-coder-problems', action='store_true', default=False) parser.add_argument('--without-calculate-problem-rating', action='store_true', default=False) parser.add_argument('--without-calculate-rating-prediction', action='store_true', default=False) parser.add_argument('--without-stage', action='store_true', default=False, help='Without update stage contests') + parser.add_argument('--without-subscriptions', action='store_true', default=False, help='Without subscriptions') parser.add_argument('--contest-id', '-cid', help='Contest id') parser.add_argument('--no-update-problems', action='store_true', default=False, help='No update problems') parser.add_argument('--is-rated', action='store_true', default=False, help='Contest is rated') @@ -208,6 +210,8 @@ def add_arguments(self, parser): parser.add_argument('--for-account', type=str, help='Events for account') parser.add_argument('--ignore-stage', action='store_true', default=False, help='Ignore stage') parser.add_argument('--disabled', action='store_true', default=False, help='Disabled module only') + parser.add_argument('--without-delete-statistics', action='store_true') + parser.add_argument('--allow-delete-statistics', action='store_true') def parse_statistic( self, @@ -223,7 +227,7 @@ def parse_statistic( random_order=False, no_update_results=False, title_regex=None, - users=None, + specific_users=None, with_stats=True, update_without_new_rating=None, force_problems=False, @@ -231,16 +235,19 @@ def parse_statistic( without_n_statistics=False, contest_id=None, query=None, - without_fill_coder_problems=False, + without_set_coder_problems=False, without_calculate_problem_rating=False, without_calculate_rating_prediction=False, without_stage=False, + without_subscriptions=False, no_update_problems=None, is_rated=None, for_account=None, ignore_stage=None, with_reparse=None, enabled=True, + without_delete_statistics=None, + allow_delete_statistics=None, ): channel_layer_handler = ChannelLayerHandler() formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%b-%d %H:%M:%S') @@ -342,6 +349,7 @@ def parse_statistic( n_statistics_created = 0 n_calculated_rating_prediction = 0 n_calculated_problem_rating = 0 + n_inherited_medals = 0 progress_bar = tqdm(contests) stages_ids = [] @@ -409,7 +417,6 @@ def parse_statistic( exception_error = None user_info_has_rating = {} to_update_socket = contest.is_running() or contest.has_hidden_results or force_socket - to_calculate_problem_rating = False event_log = EventLog.objects.create(name='parse_statistic', related=contest, status=EventStatus.IN_PROGRESS) @@ -418,25 +425,40 @@ def parse_statistic( try: now = timezone_now() is_coming = now < contest.start_time + is_archive_contest = now > contest.end_time + timedelta(days=30) and contest.parsed_time + + with_subscription = not without_subscriptions and not is_archive_contest and with_stats + prefetch_subscribed_coders = Prefetch('coders', queryset=Coder.objects.filter(n_subscribers__gt=0), + to_attr='subscribed_coders') + subscription_top_n = None + subscription_first_ac = None + if with_subscription: + subscriptions = Subscription.enabled.filter( + (Q(resource__isnull=True) | Q(resource=resource)) & + (Q(contest__isnull=True) | Q(contest=contest)) + ) + subscription_top_n = subscriptions.aggregate(n=Max('top_n'))['n'] + subscription_first_ac = subscriptions.filter(with_first_accepted=True).exists() + plugin = resource.plugin.Statistic(contest=contest) with transaction.atomic(): _ = Contest.objects.select_for_update().get(pk=contest.pk) - statistics_users = copy.deepcopy(users) - if resource.has_standings_renamed_account and users: - renamings = AccountRenaming.objects.filter(resource=resource, old_key__in=users) + statistics_users = copy.deepcopy(specific_users) + if resource.has_standings_renamed_account and specific_users: + renamings = AccountRenaming.objects.filter(resource=resource, old_key__in=specific_users) renamings = renamings.values_list('new_key', flat=True) statistics_users.extend(renamings) with REQ: statistics_by_key = {} more_statistics_by_key = {} - statistics_ids = set() + statistics_to_delete = set() has_statistics = False if not no_update_results: statistics = Statistics.objects.filter(contest=contest).select_related('account') - if users: + if specific_users: statistics = statistics.filter(account__key__in=statistics_users) for s in statistics.iterator(): addition = s.addition or {} @@ -449,8 +471,8 @@ def parse_statistic( '_no_update_n_contests': s.skip_in_stats, } has_statistics = True - statistics_ids.add(s.pk) - standings = plugin.get_standings(users=copy.deepcopy(users), + statistics_to_delete.add(s.pk) + standings = plugin.get_standings(users=copy.deepcopy(specific_users), statistics=statistics_by_key, more_statistics=more_statistics_by_key) has_standings_result = bool(standings.get('result')) @@ -465,7 +487,7 @@ def parse_statistic( if has_problem_result: continue if row.get('_no_update_n_contests') and stat['_no_update_n_contests']: - statistics_ids.remove(stat['pk']) + statistics_to_delete.remove(stat['pk']) contest_log_counter['skip_no_update'] += 1 continue row = copy.deepcopy(row) @@ -479,24 +501,32 @@ def parse_statistic( ('place' not in result_row or str(result_row['place']) == str(stat['place'])) ): if row.get('_no_update_n_contests') and stat['_no_update_n_contests']: - statistics_ids.remove(stat['pk']) + statistics_to_delete.remove(stat['pk']) contest_log_counter['skip_no_update'] += 1 result.pop(member) keep_results = standings.pop('keep_results', False) + parsed_percentage = standings.pop('parsed_percentage', None) if keep_results: + reset_place = parsed_percentage and not contest.parsed_percentage + reset_place_ids = set() result = standings.setdefault('result', {}) for member, row in statistics_by_key.items(): if member in result: continue pk = more_statistics_by_key[member]['pk'] - if pk in statistics_ids: - statistics_ids.remove(pk) + if pk in statistics_to_delete: + statistics_to_delete.remove(pk) row = copy.deepcopy(row) row['member'] = member row['_skip_update'] = True contest_log_counter['skip_update'] += 1 result[member] = row + if reset_place: + row.pop('place', None) + reset_place_ids.add(pk) + if reset_place_ids: + Statistics.objects.filter(pk__in=reset_place_ids).update(place=None, place_as_int=None) if get_item(resource, 'info.standings.skip_not_solving'): result = standings.setdefault('result', {}) @@ -509,6 +539,7 @@ def parse_statistic( for key, more_stat in more_statistics_by_key.items(): statistics_by_key[key].update(more_stat) + update_fields = [] for field, attr in ( ('url', 'standings_url'), ('contest_url', 'url'), @@ -522,7 +553,12 @@ def parse_statistic( ): if field in standings and standings[field] != getattr(contest, attr): setattr(contest, attr, standings[field]) - contest.save(update_fields=[attr]) + update_fields.append(attr) + if parsed_percentage is not None: + contest.parsed_percentage = parsed_percentage if parsed_percentage < 100 else None + update_fields.append('parsed_percentage') + if update_fields: + contest.save(update_fields=update_fields) if 'series' in standings: contest.set_series(standings.pop('series')) @@ -611,7 +647,7 @@ def parse_statistic( has_problem_stats = False results = [] - if result or users: + if result or specific_users: fields_set = set() fields_types = defaultdict(set) fields_preceding = defaultdict(set) @@ -652,7 +688,7 @@ def parse_statistic( total_problems_solving = 0 def update_problems_info(): - nonlocal last_activity, total_problems_solving, has_problem_stats + nonlocal last_activity, total_problems_solving, has_problem_stats, has_hidden problems = r.get('problems', {}) @@ -680,7 +716,8 @@ def update_problems_info(): result_str = str(v['result']) is_accepted = result_str.startswith('+') - is_hidden = result_str.startswith('?') + is_hidden_result = '?' in result_str + has_hidden |= is_hidden_result is_score = result_str and result_str[0].isdigit() result_num = as_number(v['result'], force=True) @@ -774,7 +811,7 @@ def update_problems_info(): p['n_accepted'] = p.get('n_accepted', 0) + 1 elif scored and v.get('partial'): p['n_partial'] = p.get('n_partial', 0) + 1 - elif is_hidden: + elif is_hidden_result: p['n_hidden'] = p.get('n_hidden', 0) + 1 has_problem_stats = True @@ -801,9 +838,18 @@ def update_problems_info(): contest.standings_kind = standings_kind contest.save(update_fields=['standings_kind']) + if has_hidden and not contest.has_hidden_results and is_archive_contest: + raise ExceptionParseStandings(f'archive contest = {contest} has hidden results') + if has_hidden != contest.has_hidden_results and 'has_hidden_results' not in standings: + contest.has_hidden_results = has_hidden + contest.save(update_fields=['has_hidden_results']) + if not lazy_fetch_accounts: members = [r['member'] for r in results] - accounts = resource.account_set.filter(key__in=members) + accounts = resource.account_set + if with_subscription: + accounts = accounts.prefetch_related(prefetch_subscribed_coders) + accounts = accounts.filter(key__in=members) accounts = {a.key: a for a in accounts} if contest.set_matched_coders_to_members: @@ -824,7 +870,10 @@ def update_problems_info(): skip_result = bool(r.get('_no_update_n_contests')) if lazy_fetch_accounts: - account, account_created = Account.objects.get_or_create(resource=resource, key=member) + account = Account.objects + if with_subscription: + account = account.prefetch_related(prefetch_subscribed_coders) + account, account_created = account.get_or_create(resource=resource, key=member) elif member not in accounts: account = resource.account_set.create(key=member) accounts[member] = account @@ -885,7 +934,7 @@ def update_account_time(): contest.info.get('_no_update_account_time') or skip_result or contest.end_time > now or - users + specific_users ): return @@ -940,12 +989,12 @@ def update_account_time(): user_info = next(generator) params = user_info.get('contest_addition_update_params', {}) field = user_info.get('contest_addition_update_by') or params.get('by') or 'key' # noqa - updates = user_info.get('contest_addition_update') or params.get('update') or {} # noqa + addition_update = user_info.get('contest_addition_update') or params.get('update') or {} # noqa if not isinstance(field, (list, tuple)): field = [field] user_info_has_rating[division] = False for f in field: - if getattr(contest, f) in updates: + if getattr(contest, f) in addition_update: user_info_has_rating[division] = True break except Exception: @@ -1009,6 +1058,9 @@ def update_account_info(): if account.name != name: account.name = name update_fields.append('name') + if 'deleted' in account_info: + account.deleted = account_info.pop('deleted') + update_fields.append('deleted') account.info.update(account_info) account.save(update_fields=update_fields) @@ -1079,12 +1131,15 @@ def update_problems_first_ac(): if not r.get('division'): continue p = p.setdefault(r['division'], {}) - p = p.setdefault(k, {}) - if 'first_ac' not in p: + if k not in p: continue - - if member in p['first_ac']['accounts']: + p = p.get(k, {}) + if 'first_ac' in p and member in p['first_ac']['accounts']: v['first_ac'] = True + if p.get('n_teams') and not p.get('n_accepted') and is_hidden(v): + v['try_first_ac'] = True + else: + v.pop('try_first_ac', None) def get_addition(): place = r.pop('place', None) @@ -1163,11 +1218,14 @@ def get_addition(): return defaults, addition, try_calculate_time def update_after_update_or_create(statistic, created, addition, try_calculate_time): + updates = {} + updates['has_first_ac'] = False + updated_problems = updates.setdefault('problems', []) problems = r.get('problems', {}) if not created: nonlocal calculate_time - statistics_ids.discard(statistic.pk) + statistics_to_delete.discard(statistic.pk) if contest_timeline and try_calculate_time: p_problems = statistic.addition.get('problems', {}) @@ -1189,32 +1247,38 @@ def update_after_update_or_create(statistic, created, addition, try_calculate_ti else: v['time'] = time - for p in problems.values(): - p_result = p.get('result', '') - if isinstance(p_result, str) and '?' in p_result: - nonlocal has_hidden - has_hidden = True + if force_socket: + updated_statistics_ids.append(statistic.pk) previous_problems = stat.get('problems', {}) for k, problem in problems.items(): - if statistic.pk in updated_statistics_ids: - break + verdict = problem.get('verdict') previous_problem = previous_problems.get(k, {}) - if ( - problem.get('result') != previous_problem.get('result') or - problem.get('time') != previous_problem.get('time') or - force_socket - ): - contest_log_counter['updated_statistics_problem'] += 1 - updated_statistics_ids.append(statistic.pk) - - if str(statistic.place) != str(stat.get('place')): - contest_log_counter['updated_statistics_place'] += 1 + previous_verdict = previous_problem.get('verdict') + same_result = problem.get('result') == previous_problem.get('result') + same_verdict = not verdict or not previous_verdict or verdict == previous_verdict + if same_result and same_verdict: + continue + if not same_result and problem.get('first_ac'): + updates['has_first_ac'] = True + updated_problems.append(k) + if statistic.pk in updated_statistics_ids: + continue + contest_log_counter['updated_statistics_problem'] += 1 updated_statistics_ids.append(statistic.pk) + for field, lhs, rhs in ( + ('place', statistic.place, stat.get('place')), + ('score', statistic.solving, stat.get('score')), + ): + if lhs != rhs and statistic.pk not in updated_statistics_ids: + contest_log_counter[f'updated_statistics_{field}'] += 1 + updated_statistics_ids.append(statistic.pk) + if try_calculate_time: statistic.addition = addition statistic.save() + return updates @suppress_db_logging_context() def update_submissions(statistic, result_submissions): @@ -1226,7 +1290,7 @@ def update_submissions(statistic, result_submissions): contest.save(update_fields=['has_submissions']) statistic_problems = statistic.addition.get('problems', {}) - updated_problems = False + updated_submission_problems = False submission_ids = set() for result_submission in result_submissions: language = Language.cached_get(result_submission['language']) @@ -1337,10 +1401,10 @@ def update_submissions(statistic, result_submissions): for field, value in fields_values: if value is not None and statistic_problem.get(field) != value: statistic_problem[field] = value - updated_problems = True + updated_submission_problems = True if update_problems_values(statistic.addition): - updated_problems = True - if updated_problems: + updated_submission_problems = True + if updated_submission_problems: statistic.save(update_fields=['addition']) if os.environ.get('DELETE_SUBMISSIONS'): extra = Submission.objects.filter(statistic=statistic) @@ -1401,12 +1465,60 @@ def set_matched_coders_to_members(statistic): if to_update: statistic.save(update_fields=['addition']) + def process_subscriptions(statistic, updates): + if skip_result or not with_subscription: + return + if addition.get('_skip_subscription') or not updates['problems']: + return + + with_top_n = (subscription_top_n and statistic.place_as_int and + statistic.place_as_int <= subscription_top_n) + with_first_ac = updates['has_first_ac'] and subscription_first_ac + subscribed_coders = getattr(account, 'subscribed_coders', None) + if ( + not account.n_subscribers and + not subscribed_coders and + not with_top_n and + not with_first_ac + ): + return + + subscription_message = compose_message_by_problems( + updates['problems'], + statistic=statistic, + previous_addition=stat, + contest_or_problems=standings_problems, + ) + + subscriptions_filter = Q() + if account.n_subscribers: + subscriptions_filter |= Q(accounts=account) + if subscribed_coders: + subscriptions_filter |= Q(coders__in=subscribed_coders) + if with_top_n: + subscriptions_filter |= Q(top_n__gte=statistic.place_as_int) + if with_first_ac: + subscriptions_filter |= Q(with_first_accepted=True) + subscriptions_filter = ( + subscriptions_filter + & (Q(resource__isnull=True) | Q(resource=resource)) + & (Q(contest__isnull=True) | Q(contest=contest)) + ) + subscriptions = Subscription.enabled.filter(subscriptions_filter) + already_sent = set() + for subscription in subscriptions: + if subscription.notification_key in already_sent: + continue + already_sent.add(subscription.notification_key) + subscription.send(message=subscription_message, contest=contest) + contest_log_counter['statistics_subscription'] += 1 + update_addition_fields() update_account_time() update_account_info() update_stat_info() update_problems_values(r) - if not users: + if not specific_users: update_problems_first_ac() defaults, addition, try_calculate_time = get_addition() @@ -1421,7 +1533,8 @@ def set_matched_coders_to_members(statistic): if statistic_created: contest_log_counter['statistics_created'] += 1 - update_after_update_or_create(statistic, statistic_created, addition, try_calculate_time) + updates = update_after_update_or_create(statistic, statistic_created, addition, + try_calculate_time) update_submissions(statistic, result_submissions) @@ -1431,33 +1544,9 @@ def set_matched_coders_to_members(statistic): if contest.set_matched_coders_to_members: set_matched_coders_to_members(statistic) - has_subscription_update = ( - account.is_subscribed - and not skip_result - and not addition.get('_skip_subscription') - and ( - statistic_created - or stat.get('score', 0) < statistic.solving + EPS - ) - ) - if has_subscription_update: - subscriptions_filter = Q(resource__isnull=True) | Q(resource=resource) - subscriptions_filter &= Q(contest__isnull=True) | Q(contest=contest) - subscriptions_filter &= ( - Q(account=account) - | Q(coder_chat__coders__account=account) - | Q(coder_list__values__account=account) - | Q(coder_list__values__coder__account=account) - ) - subscriptions = Subscription.objects.filter(subscriptions_filter) - for subscription in subscriptions: - pass - - if not users: - if has_hidden != contest.has_hidden_results and 'has_hidden_results' not in standings: - contest.has_hidden_results = has_hidden - contest.save(update_fields=['has_hidden_results']) + process_subscriptions(statistic, updates) + if not specific_users: for field, values in problems_values.items(): if values: field = field.strip('_') @@ -1478,11 +1567,14 @@ def set_matched_coders_to_members(statistic): fields.remove(rating_field) fields.append(rating_field) - if statistics_ids: - first = Statistics.objects.filter(pk__in=statistics_ids).first() + if statistics_to_delete and not without_delete_statistics: + if is_archive_contest and not allow_delete_statistics: + raise ExceptionParseStandings(f'archive contest = {contest} ' + f'has statistics to delete') + first = Statistics.objects.filter(pk__in=statistics_to_delete).first() if first: self.logger.info(f'First deleted: {first}, account = {first.account}') - delete_info = Statistics.objects.filter(pk__in=statistics_ids).delete() + delete_info = Statistics.objects.filter(pk__in=statistics_to_delete).delete() self.logger.info(f'Delete info: {delete_info}') progress_bar.set_postfix(deleted=str(delete_info)) n_deleted, _ = delete_info @@ -1577,20 +1669,16 @@ def set_matched_coders_to_members(statistic): if to_update_socket: update_standings_socket(contest, updated_statistics_ids) - to_calculate_problem_rating = ( - resource.has_problem_rating and - contest.end_time < now and - not contest.has_hidden_results and - not standings.get('timing_statistic_delta') - ) - progress_bar.set_postfix(n_fields=len(fields), n_updated=len(updated_statistics_ids)) else: if standings_problems is not None and standings_problems and not no_update_problems: standings_problems = plugin.merge_dict(standings_problems, contest.info.get('problems')) - update_problems(contest, problems=standings_problems, force=force_problems or not users) + update_problems(contest, problems=standings_problems, force=force_problems) - if not users: + if contest_log_counter.get('statistics_subscription'): + send_messages() + + if not specific_users: timing_delta = None if contest.full_duration < resource.module.long_contest_idle: timing_delta = standings.get('timing_statistic_delta', timing_delta) @@ -1598,7 +1686,7 @@ def set_matched_coders_to_members(statistic): timing_delta = parse_info.get('timing_statistic_delta', timing_delta) if updated_statistics_ids and contest.end_time < now < contest.end_time + timedelta(hours=1): timing_delta = timing_delta or timedelta(minutes=20) - if has_hidden and contest.end_time < now < contest.end_time + timedelta(days=1): + if contest.has_hidden_results and contest.end_time < now < contest.end_time + timedelta(days=1): timing_delta = timing_delta or timedelta(minutes=60) if wait_rating and not has_statistics and results and 'days' in wait_rating: timing_delta = timing_delta or timedelta(days=wait_rating['days']) / 10 @@ -1613,7 +1701,6 @@ def set_matched_coders_to_members(statistic): contest.save() else: without_calculate_rating_prediction = True - to_calculate_problem_rating = False action = standings.get('action') if action is not None: @@ -1635,31 +1722,53 @@ def set_matched_coders_to_members(statistic): contest.url = args[0] contest.save() - if not contest.statistics_update_required: - if resource.rating_prediction and not without_calculate_rating_prediction and contest.pk: + if not contest.statistics_update_required and contest.pk: + if resource.rating_prediction and not without_calculate_rating_prediction and not specific_users: self.logger.info(f'Calculate rating prediction for contest = {contest}') call_command('calculate_rating_prediction', contest=contest.pk) contest.refresh_from_db() n_calculated_rating_prediction += 1 - if to_calculate_problem_rating and not without_calculate_problem_rating and contest.pk: + if ( + resource.has_problem_rating and contest.is_finalized() and not without_calculate_problem_rating + and not specific_users + ): self.logger.info(f'Calculate problem rating for contest = {contest}') call_command('calculate_problem_rating', contest=contest.pk, force=force_problems) contest.refresh_from_db() n_calculated_problem_rating += 1 - if not without_fill_coder_problems and contest.pk: - if users: - users_qs = Account.objects.filter(resource=resource, key__in=users) + if contest.is_finalized(): + related_contests = [contest] + for related in contest.related_set.select_related('resource', 'related').all(): + related_contests.append(related) + for orig_contest in related_contests: + related_contest = orig_contest.related + if not related_contest: + continue + if not orig_contest.resource.has_inherit_medals_to_related: + continue + if not orig_contest.is_finalized(): + continue + if not orig_contest.with_medals or related_contest.with_medals: + continue + self.logger.info(f'Inherit medals to related contest = {related_contest}' + f', from contest = {orig_contest}') + related_contest.inherit_medals(orig_contest) + n_inherited_medals += 1 + + if not without_set_coder_problems: + if specific_users: + users_qs = Account.objects.filter(resource=resource, key__in=specific_users) users_coders = Coder.objects.filter(Exists(users_qs.filter(pk=OuterRef('account')))) users_coders = users_coders.values_list('username', flat=True) if users_coders: - self.logger.info(f'Fill coder problems for contest = {contest}' + self.logger.info(f'Set coder problems for contest = {contest}' f', coders = {users_coders}') - call_command('fill_coder_problems', contest=contest.pk, coders=users_coders) + call_command('set_coder_problems', contest=contest.pk, coders=users_coders) else: - self.logger.info(f'Fill coder problems for contest = {contest}') - call_command('fill_coder_problems', contest=contest.pk) + self.logger.info(f'Set coder problems for contest = {contest}') + call_command('set_coder_problems', contest=contest.pk) if has_standings_result: count += 1 @@ -1678,7 +1787,7 @@ def set_matched_coders_to_members(statistic): if stop_on_error: print(colored_format_exc()) - if not users: + if not specific_users: module = resource.module delay = module.max_delay_after_end if contest.statistics_update_required or contest.end_time < now or not module.long_contest_divider: @@ -1715,7 +1824,7 @@ def set_matched_coders_to_members(statistic): if contest_log_counter: messages += [f'log_counter = {dict(contest_log_counter)}'] event_log.update_status(status=status, message='\n\n'.join(messages)) - if users: + if specific_users: event_log.delete() self.logger.info(f'log_counter = {dict(contest_log_counter)}') if error_counter: @@ -1723,12 +1832,12 @@ def set_matched_coders_to_members(statistic): channel_layer_handler.send_done(done=parsed) @lru_cache(maxsize=None) - def update_stage(stage): + def advanced_update_stage(stage): exclude_stages = stage.score_params.get('advances', {}).get('exclude_stages', []) ret = stage.pk in stages_ids if exclude_stages: for s in Stage.objects.filter(pk__in=exclude_stages): - if update_stage(s): + if advanced_update_stage(s): ret = True if ret: try: @@ -1736,7 +1845,7 @@ def update_stage(stage): related=stage, status=EventStatus.IN_PROGRESS) channel_layer_handler.set_contest(stage.contest) - stage.update() + update_stage(stage) channel_layer_handler.send_done(done=True) event_log.update_status(EventStatus.COMPLETED) except Exception as e: @@ -1750,7 +1859,7 @@ def update_stage(stage): if without_stage: self.logger.info(f'Skip stage: {stage}') else: - update_stage(stage) + advanced_update_stage(stage) if resource_event_log: resource_event_log.delete() @@ -1761,6 +1870,8 @@ def update_stage(stage): self.logger.info(f'Number of calculate rating problem: {n_calculated_problem_rating} of {total}') if n_calculated_rating_prediction: self.logger.info(f'Number of calculate rating prediction: {n_calculated_rating_prediction} of {total}') + if n_inherited_medals: + self.logger.info(f'Number of inherited medals: {n_inherited_medals} of {total}') self.logger.info(f'Number of updated account time: {n_account_time_update}') self.logger.info(f'Number of created statistics: {n_statistics_created} of {n_statistics_total}') @@ -1823,7 +1934,7 @@ def handle(self, *args, **options): before_date=args.before_date, after_date=args.after_date, title_regex=args.event, - users=args.users, + specific_users=args.users, with_stats=not args.no_stats, update_without_new_rating=args.update_without_new_rating, force_problems=args.force_problems, @@ -1831,14 +1942,17 @@ def handle(self, *args, **options): without_n_statistics=args.without_n_statistics, contest_id=args.contest_id, query=args.query, - without_fill_coder_problems=args.without_fill_coder_problems, + without_set_coder_problems=args.without_set_coder_problems, without_calculate_problem_rating=args.without_calculate_problem_rating, without_calculate_rating_prediction=args.without_calculate_rating_prediction, without_stage=args.without_stage, + without_subscriptions=args.without_subscriptions, no_update_problems=args.no_update_problems, is_rated=args.is_rated, for_account=args.for_account, ignore_stage=args.ignore_stage, with_reparse=args.reparse, enabled=not args.disabled, + without_delete_statistics=args.without_delete_statistics, + allow_delete_statistics=args.allow_delete_statistics, ) diff --git a/src/ranking/management/modules/acm_bsu.py b/src/ranking/management/modules/acm_bsu.py index aa406ec6..9b7a169a 100644 --- a/src/ranking/management/modules/acm_bsu.py +++ b/src/ranking/management/modules/acm_bsu.py @@ -4,7 +4,6 @@ import collections import re from datetime import timedelta -from pprint import pprint from django.utils.timezone import now @@ -34,7 +33,10 @@ def get_standings(self, users=None, statistics=None, **kwargs): if e.code == 403: raise ExceptionParseStandings('Forbidden') raise e - table = parsed_table.ParsedTable(html=page, xpath="//table[@class='ir-contest-standings']//tr") + try: + table = parsed_table.ParsedTable(html=page, xpath="//table[@class='ir-contest-standings']//tr") + except StopIteration: + raise ExceptionParseStandings('Not found table with standings') problems_info = collections.OrderedDict() has_plus = False for r in table: @@ -137,5 +139,3 @@ def get_standings(self, users=None, statistics=None, **kwargs): 'problems_time_format': '{H}:{m:02d}', } return standings - - diff --git a/src/ranking/management/modules/algorithm_yandex.py b/src/ranking/management/modules/algorithm_yandex.py index 09215a84..b627757c 100644 --- a/src/ranking/management/modules/algorithm_yandex.py +++ b/src/ranking/management/modules/algorithm_yandex.py @@ -14,4 +14,6 @@ def get_standings(self, *args, **kwargs): if 'medals' not in standings.get('options', {}) and 'medals' not in self.info.get('standings', {}): options = standings.setdefault('options', {}) options['medals'] = [{'name': name, 'count': 1} for name in ('gold', 'silver', 'bronze')] + if re.search(r'\balgorithm\b', self.name, re.I): + standings['series'] = 'yandex-algorithm' return standings diff --git a/src/ranking/management/modules/algotester.py b/src/ranking/management/modules/algotester.py index 71d55209..e95ccd74 100644 --- a/src/ranking/management/modules/algotester.py +++ b/src/ranking/management/modules/algotester.py @@ -80,6 +80,8 @@ def proccess_data(data): for row in data['rows']: contestant = row.pop('Contestant') url = contestant.pop('Url') + if url is None: + continue match = re.search('(?P<type>Account|Team)/Display/(?P<key>[0-9]+)', url) handle = match.group('type').lower() + match.group('key') r = result.setdefault(handle, OrderedDict()) diff --git a/src/ranking/management/modules/atcoder.py b/src/ranking/management/modules/atcoder.py index 06ff0d67..9c1d4c32 100644 --- a/src/ranking/management/modules/atcoder.py +++ b/src/ranking/management/modules/atcoder.py @@ -403,10 +403,10 @@ def get_problem_full_score(info): if users is not None and handle not in users: continue r = result.setdefault(handle, collections.OrderedDict()) + account_info = r.setdefault('info', {}) r['member'] = handle - if row.pop('UserIsDeleted', None): - r['action'] = 'delete' - continue + if (v := row.pop('UserIsDeleted', None)) is not None: + account_info['deleted'] = v r['place'] = row.pop('Rank') total_result = row.pop('TotalResult') @@ -461,7 +461,7 @@ def get_problem_full_score(info): row['AtcoderRank'] = row.pop('AtCoderRank') rating = row.pop('Rating', None) if rating is not None: - r['info'] = {'rating': rating} + account_info['rating'] = rating old_rating = row.pop('OldRating', None) for k, v in sorted(row.items()): @@ -495,11 +495,10 @@ def get_problem_full_score(info): if users is not None and handle not in users: continue r = result.setdefault(handle, collections.OrderedDict()) + account_info = r.setdefault('info', {}) r['member'] = handle - if row.pop('UserIsDeleted', None): - r['action'] = 'delete' - continue - + if (v := row.pop('UserIsDeleted', None)) is not None: + account_info['deleted'] = v r['country'] = row.pop('Country') if 'UserName' in row: r['name'] = row.pop('UserName') @@ -688,7 +687,7 @@ def get_source_code(contest, problem): filepath_proxies=filepath_proxies, connect=lambda req: req.get(problem['url']), ) as req: - _, page = req.proxer.get_connect_ret() + page = req.proxer.get_connect_ret() match = re.search('<pre[^>]*id="submission-code"[^>]*>(?P<source>[^<]*)</pre>', page) if not match: diff --git a/src/ranking/management/modules/codechef.py b/src/ranking/management/modules/codechef.py index b2897fc2..1bc41185 100644 --- a/src/ranking/management/modules/codechef.py +++ b/src/ranking/management/modules/codechef.py @@ -215,7 +215,7 @@ def get_standings(self, users=None, statistics=None, **kwargs): handle = d.pop('user_handle') d.pop('html_handle', None) problems_status = d.pop('problems_status') - if d['score'] < 1e-9 and not problems_status: + if handle is None or d['score'] < 1e-9 and not problems_status: LOG.warning(f'Skip handle = {handle}: {d}') continue row = result.setdefault(handle, OrderedDict()) diff --git a/src/ranking/management/modules/codeforces.py b/src/ranking/management/modules/codeforces.py index af6132cd..ee159db5 100644 --- a/src/ranking/management/modules/codeforces.py +++ b/src/ranking/management/modules/codeforces.py @@ -23,7 +23,7 @@ API_KEYS = conf.CODEFORCES_API_KEYS DEFAULT_API_KEY = API_KEYS[API_KEYS['__default__']] -SUBDOMAIN = '' +SUBDOMAIN = 'mirror.' def api_query( @@ -85,7 +85,7 @@ def api_query( continue ret = {'status': str(e)} break - + ret.setdefault('status', 'EMPTY') return ret @@ -488,9 +488,9 @@ def parse_points_info(points_info): if not users: data = api_query(method='contest.ratingChanges', params=params, api_key=self.api_key) - if data.get('status') not in ['OK', 'FAILED']: + if data['status'] not in ['OK', 'FAILED']: LOG.warning(f'Missing rating changes = {data}') - if data and data.get('status') == 'OK': + if data and data['status'] == 'OK': for row in data['result']: if str(row.pop('contestId')) != self.key: continue @@ -525,7 +525,7 @@ def parse_points_info(points_info): submissions = [] for params in array_params: data = api_query('contest.status', params=params, api_key=self.api_key) - if data.get('status') not in ['OK', 'FAILED']: + if data['status'] not in ['OK', 'FAILED']: raise ExceptionParseStandings(data) if data['status'] == 'OK': submissions.extend(data.pop('result')) @@ -647,7 +647,7 @@ def get_users_infos(users, resource=None, accounts=None, pbar=None): and (match := re.search('handles: User with handle (?P<handle>.*) not found', data['comment'])) ): handle = match.group('handle') - location = REQ.geturl(f'https://codeforces.com/profile/{handle}') + location = REQ.geturl(f'https://{SUBDOMAIN}codeforces.com/profile/{handle}') index = users.index(handle) if urlparse(location).path.rstrip('/'): target = location.rstrip('/').split('/')[-1] diff --git a/src/ranking/management/modules/common/__init__.py b/src/ranking/management/modules/common/__init__.py index c7ebdf06..600275b7 100644 --- a/src/ranking/management/modules/common/__init__.py +++ b/src/ranking/management/modules/common/__init__.py @@ -17,7 +17,10 @@ def create_requester(): - req = requester(cookie_filename='sharedfiles/cookies.txt') + req = requester( + cookie_filename='sharedfiles/cookies.txt', + proxy_filepath='sharedfiles/proxy', + ) req.caching = 'REQUESTER_CACHING' in os.environ req.time_out = 45 req.debug_output = 'REQUESTER_DEBUG' in os.environ diff --git a/src/ranking/management/modules/cphof.py b/src/ranking/management/modules/cphof.py index ec0a6f11..9b198522 100644 --- a/src/ranking/management/modules/cphof.py +++ b/src/ranking/management/modules/cphof.py @@ -105,7 +105,7 @@ def find_related(statistics): ignore_n_statistics = False ignore_title = None for mapping in host_mapping: - if re.search(mapping['regex'], host): + if re.search(mapping['regex'], host, re.IGNORECASE): host = mapping['host'] ignore_title = mapping.get('ignore_title') ignore_n_statistics = mapping.get('ignore_n_statistics', ignore_n_statistics) diff --git a/src/ranking/management/modules/facebook.py b/src/ranking/management/modules/facebook.py index 53f1fc3d..911a8a60 100644 --- a/src/ranking/management/modules/facebook.py +++ b/src/ranking/management/modules/facebook.py @@ -5,46 +5,52 @@ import re from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor as PoolExecutor -from datetime import datetime +from datetime import datetime, timedelta +from threading import Lock from time import sleep import pytz import requests import tqdm +from django.utils.timezone import now +from ratelimiter import RateLimiter # from ranking.management.modules import conf from ranking.management.modules.common import LOG, REQ, BaseModule from ranking.management.modules.excepts import ExceptionParseStandings +def is_rate_limit_error(e): + return bool(re.search('rate limit exceeded', str(e), re.I)) + + class Statistic(BaseModule): API_GRAPH_URL_ = 'https://www.facebook.com/api/graphql/' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko' - self.headers = {'User-Agent': user_agent} - # facebook_page = REQ.get('https://facebook.com/', headers=self.headers) # noqa: F841 - # form = REQ.form(action='/login/') - # if form: - # form['post'].pop('sign_up', None) - # data = { - # 'email': conf.FACEBOOK_USERNAME, - # 'pass': conf.FACEBOOK_PASSWORD, - # } - # signin_page = REQ.submit_form(data=data, form=form, headers=self.headers) # noqa: F841 - # form = REQ.form(action='/login/') - # if form and 'validate-password' in form['url']: - # REQ.submit_form(data=data, form=form, headers=self.headers) + with open('sharedfiles/resource/facebook/headers.json') as file: + self.headers = json.load(file) def get_standings(self, users=None, statistics=None, **kwargs): + n_proxies = 50 + req = REQ.duplicate(cookie_filename='sharedfiles/resource/facebook/cookies.txt') + is_final = bool(re.search(r'\bfinals?\b', self.name, re.IGNORECASE)) - page = REQ.get(self.standings_url, headers=self.headers) + page = req.get(self.standings_url, headers=self.headers) matches = re.finditer(r'\["(?P<name>[^"]*)",\[\],{"token":"(?P<token>[^"]*)"', page) tokens = {} for match in matches: tokens[match.group('name').lower()] = match.group('token') + req = req.with_proxy( + time_limit=10, + n_limit=n_proxies, + filepath_proxies='sharedfiles/resource/facebook/proxies', + attributes={'n_attempts': n_proxies}, + ) + lock = Lock() + def query(name, variables): params = { 'fb_dtsg': tokens.get('dtsginitialdata', ''), @@ -56,10 +62,14 @@ def query(name, variables): 'doc_id': self.info['_scoreboard_ids'][name], } - n_attempts = 3 + n_attempts = 1 + n_rate_limit_exceeded = 3 seen = set() - for idx in range(n_attempts): - ret = REQ.get( + attempt = 0 + while True: + attempt += 1 + + ret = req.get( self.API_GRAPH_URL_, post=params, headers={'Accept-Language': 'en-US,en;q=1.0', **self.headers}, @@ -78,19 +88,28 @@ def query(name, variables): except Exception as e: msg = f'Exception on query {name} = {e}' - is_last = idx + 1 == n_attempts - if is_last: - raise ExceptionParseStandings(msg) if msg not in seen: LOG.warning(msg) seen.add(msg) - sleep(idx) - variables = {'id': self.key, 'force_limited_data': False, 'show_all_submissions': False} + if is_rate_limit_error(msg): + req.proxy_fail(force=True) + if n_rate_limit_exceeded and is_rate_limit_error(msg): + n_rate_limit_exceeded -= 1 + elif n_attempts: + n_attempts -= 1 + else: + raise ExceptionParseStandings(msg) + + with lock: + sleep(attempt) + + variables = {'id': self.key, 'force_limited_data': False, 'show_all_submissions': False, + 'should_include_scoreboard': True} scoreboard_data = query('CodingCompetitionsContestScoreboardQuery', variables) def get_advance(): - advancement = scoreboard_data['data']['contest'].get('advancement_requirement_text') + advancement = contest_data.get('advancement_requirement_text') if advancement: match = re.search('top (?P<place>[0-9]+) contestants', advancement) if match: @@ -107,9 +126,9 @@ def get_advance(): threshold = int(match.group('count')) return {'filter': [{'threshold': threshold, 'operator': 'ge', 'field': '_n_solved'}]} - advancement = scoreboard_data['data']['contest'].get('advancement_mode') + advancement = contest_data.get('advancement_mode') if advancement: - threshold = scoreboard_data['data']['contest']['advancement_value'] + threshold = contest_data['advancement_value'] if advancement == 'RANK': return {'filter': [{'threshold': threshold, 'operator': 'le', 'field': 'place'}]} if advancement == 'SCORE': @@ -119,8 +138,15 @@ def get_advance(): return {} + def get_contest(scoreboard_data): + data = scoreboard_data['data'] + for k in 'contest', 'fetch__CodingContest': + if k in data: + return data[k] + problems_info = OrderedDict() - for problem_set in scoreboard_data['data']['contest']['ordered_problem_sets']: + contest_data = get_contest(scoreboard_data) + for problem_set in contest_data['ordered_problem_sets']: for problem in problem_set['ordered_problems_with_display_indices']: code = str(problem['problem']['id']) info = { @@ -133,37 +159,55 @@ def get_advance(): problems_info[info['code']] = info limit = 50 - total = scoreboard_data['data']['contest']['entrant_performance_summaries']['count'] + total = contest_data['entrant_performance_summaries']['count'] + info_paging_offset = self.info.get('_paging_offset', 0) if statistics else 0 + paging_offset = info_paging_offset + parsed_percentage = None has_hidden = False result = OrderedDict() if users or users is None: has_users_filter = bool(users) - users = set(users) if users else None + if has_users_filter: + users = set(users) + paging_offset = 0 + with PoolExecutor(max_workers=3) as executor: stop = False + @RateLimiter(max_calls=1, period=1) def fetch_page(page): if stop: return - data = query('CCEScoreboardQuery', { - 'id': self.key, - 'start': page * limit, - 'count': limit, - 'friends_only': False, - 'force_limited_data': False, - 'country_filter': None, - 'show_all_submissions': False, - 'substring_filter': '', - }) + try: + data = query('CCEScoreboardQuery', { + 'id': self.key, + 'start': page * limit, + 'count': limit, + 'friends_only': False, + 'force_limited_data': False, + 'country_filter': None, + 'show_all_submissions': False, + 'substring_filter': '', + }) + except Exception as e: + if not is_rate_limit_error(e): + LOG.error(f'Fetch page exception = {e}') + return return data n_page = (total + limit - 1) // limit - for data in tqdm.tqdm(executor.map(fetch_page, range(n_page)), total=n_page, desc='paging'): + pages = list(range(paging_offset, n_page)) + tqdm_iterator = tqdm.tqdm(executor.map(fetch_page, pages), total=len(pages), desc='paging') + process_pages = set() + for page, data in zip(pages, tqdm_iterator): + if data is None: + stop = True + break if not data: continue - - for row in data['data']['contest']['entrant_performance_summaries']['nodes']: + process_pages.add(page) + for row in get_contest(data)['entrant_performance_summaries']['nodes']: row.update(row.pop('entrant')) handle = row.pop('id') if has_users_filter and handle not in users: @@ -237,13 +281,30 @@ def fetch_page(page): if not problems and not is_final: result.pop(handle) continue + while paging_offset in process_pages: + paging_offset += 1 + parsed_percentage = paging_offset * 100. / n_page if n_page else None + if paging_offset >= n_page: + paging_offset = 0 + + req.__exit__() standings = { 'result': result, 'problems': list(problems_info.values()), 'advance': get_advance(), 'has_hidden': has_hidden, + 'keep_results': info_paging_offset or paging_offset, + 'info_fields': ['_paging_offset'], } + if not has_users_filter: + standings['_paging_offset'] = paging_offset + standings['parsed_percentage'] = parsed_percentage + if ( + parsed_percentage and parsed_percentage < 100 and + self.end_time < now() < self.end_time + timedelta(days=1) + ): + standings['timing_statistic_delta'] = timedelta(minutes=15) if is_final: standings['series'] = 'FHC' diff --git a/src/ranking/management/modules/icpc_baylor.py b/src/ranking/management/modules/icpc_baylor.py index 22439719..7e866bb7 100644 --- a/src/ranking/management/modules/icpc_baylor.py +++ b/src/ranking/management/modules/icpc_baylor.py @@ -48,8 +48,8 @@ def names_iou(name1, name2): iou = max(list_iou, str_iou) n = min(len(canonized_name1), len(canonized_name2)) - prefix_iou = list_string_iou(canonized_name1[:n], canonized_name2[:n]) - suffix_iou = list_string_iou(canonized_name1[-n:], canonized_name2[-n:]) + prefix_iou = list_string_iou(canonized_name1[:n], canonized_name2[:n]) * 0.99 + suffix_iou = list_string_iou(canonized_name1[-n:], canonized_name2[-n:]) * 0.95 iou = max(iou, prefix_iou, suffix_iou) return iou @@ -433,7 +433,7 @@ def get(key, index): if is_icpc_api_standings_url: page = re.sub(r'</table>\s*<table>\s*(<tr[^>]*>\s*<t[^>]*>)', r'\1', page, flags=re.I) - regex = '''(?:<table[^>]*(?:id=["']standings|class=["']scoreboard)[^>]*>|"content":"[^"]*<table[^>]*>|<table[^>]*class="[^"]*(?:table[^"]*){3}"[^>]*>).*?</table>''' # noqa + regex = '''(?:<table[^>]*(?:id=["']standings|class=["'][^"']*scoreboard)[^>]*>|"content":"[^"]*<table[^>]*>|<table[^>]*class="[^"]*(?:table[^"]*){3}"[^>]*>).*?</table>''' # noqa match = re.search(regex, page, re.DOTALL) if match: html_table = match.group(0) @@ -489,9 +489,9 @@ def get(key, index): logo = urljoin(standings_url, src) row.setdefault('info', {}).setdefault('logo', logo) for el in vs: - region = el.column.node.xpath('.//*[@class="badge badge-warning"]') + region = el.column.node.xpath('.//*[contains(@class, "badge")]') if region: - region = ''.join([s.strip() for s in region[0].xpath('text()')]) + region = ' '.join([s.strip() for s in region[0].xpath('text()')]) if region: if is_regional: if region.lower() == 'ineligible': @@ -586,6 +586,25 @@ def get(key, index): region = ''.join([s.strip() for s in prv.xpath('text()')]) row['region'] = region + regex = '''<table[^>]*class="[^"]*table-sm[^"]*"[^>]*>.*?</table>''' + tables = re.findall(regex, page, re.DOTALL) + for table in tables: + table = parsed_table.ParsedTable(table) + values = [] + for r in table: + for k, v in r.items(): + values.append(k) + values.append(v.value) + if len(values) > 4: + break + if len(values) == 4 and values[:2] == ['Name', 'Category']: + name, region = values[2:] + member = f'{name} {season}' + if member not in result: + logger.warning(f'Not found member = {member} for region = {region}') + continue + result[member]['region'] = region + for team, row in result.items(): if statistics and team in statistics: row.pop('info', None) @@ -603,6 +622,8 @@ def get(key, index): if not stat: continue for k, v in stat.items(): + if k in ['medal']: + continue if k not in row: hidden_fields.add(k) row[k] = v @@ -692,7 +713,6 @@ def add_region(name, region, team): best_region = region best_team = team best_name = row['name'] - print(name) if best_name is None: break logger.info(f'max iou = {max_iou:.3f}, best_name = {best_name}') @@ -817,11 +837,57 @@ def add_team(university, handles): logger_func = logger.info add_team(best_name, handles) logger_func(f'best_iou = {best_iou:.3f}, best_name = {best_name}, university = {university}') + if 'use_zibada_list' in self.info and os.environ.get('USE_ZIBADA_LIST'): + zibada_info = self.info['use_zibada_list'] + data = REQ.get(zibada_info['url']) + data = re.sub(r'^[\w\d]+\s*=\s*', '', data) + data = json.loads(data) + for team in data: + name, handles = team + name = html.unescape(name) + member = f'{name} {season}' + if member not in result: + logger.warning(f'Not found team = {member}') + continue + row = result[member] + members = row.setdefault('_members', []) + accounts = {m['account'] for m in members} + for handle in handles: + handle = handle.split(':')[-1] + if handle in accounts: + continue + members.append({ + 'account': handle, + 'resource': zibada_info['resource'], + 'without_country': True, + }) + if len(members) > 3: + logger.warning(f'Too many members: {members}, team = {member}') event_feed = get_item(self.info, 'standings._event_feed') if event_feed: self._parse_event_feed(event_feed, result, problems_info) + info_external_urls = get_item(self.info, 'standings.external_urls', []) + info_external_urls_set = set(i['url'] for i in info_external_urls) + + external_urls = [ + f'https://icpc.kimden.online/wf/{year}/', + f'https://zibada.guru/finals/{year}/', + ] + for external_url in external_urls: + if external_url in info_external_urls_set: + continue + try: + REQ.head(external_url) + except FailOnGetResponse: + continue + info_external_urls.append({ + 'url': external_url, + 'name': urlparse(external_url).hostname, + }) + options['external_urls'] = info_external_urls + standings = { 'result': result, 'url': icpc_standings_url if is_icpc_api_standings_url else standings_url, diff --git a/src/ranking/management/modules/leetcode.py b/src/ranking/management/modules/leetcode.py index ce8aa575..a57e41cb 100644 --- a/src/ranking/management/modules/leetcode.py +++ b/src/ranking/management/modules/leetcode.py @@ -78,7 +78,7 @@ def _get(self, *args, req=None, n_addition_attempts=20, **kwargs): headers.setdefault(key, value) kwargs['with_curl'] = req.proxer is None and 'post' not in kwargs - kwargs['curl_cookie_file'] = os.path.join(os.path.dirname(__file__), '.leetcode.cookies') + kwargs['curl_cookie_file'] = 'sharedfiles/resource/leetcode/cookies.txt' kwargs['with_referer'] = False # if not getattr(self, '_authorized', None) and getattr(conf, 'LEETCODE_COOKIES', False): @@ -123,6 +123,7 @@ def fetch_standings_page(page): url = api_ranking_url_format.format(page + 1) content = Statistic._get(url, ignore_codes={404}, n_attempts=3) data = json.loads(content) + data['_page'] = page + 1 return data data = fetch_standings_page(0) @@ -184,21 +185,25 @@ def fetch_ranking(domain, region, n_page=None): raise e if not data: return - for p in data['questions']: - if str(p['question_id']) not in problems_info: - return + if 'questions' in data: + for p in data['questions']: + if str(p['question_id']) not in problems_info: + return per_page = len(data['total_rank']) n_page = n_page or (data['user_num'] - 1) // per_page + 1 if users and more_statistics: - pages = set() + pages = [] for more_stat in more_statistics.values(): place = as_number(more_stat['place'], force=True) if place is None: raise ExceptionParseStandings(f'Invalid place {more_stat["place"]}') - page = (place - 1) // per_page - pages.add(page) + base_page = (place - 1) // per_page + for page_delta in (0, -1, +1, -2, +2): + page = base_page + page_delta + if page not in pages: + pages.append(page) n_page = len(pages) else: pages = range(n_page) @@ -212,6 +217,8 @@ def fetch_ranking(domain, region, n_page=None): if stop_fetch_standings: break n_added = 0 + if 'submissions' not in data: + data['submissions'] = [r.pop('submissions') for r in data['total_rank']] for row, submissions in zip(data['total_rank'], data['submissions']): handle = row.pop('user_slug').lower() data_region = row.pop('data_region').lower() @@ -220,8 +227,8 @@ def fetch_ranking(domain, region, n_page=None): already_added = member in result if users is not None and member not in users: continue - row.pop('contest_id') - row.pop('global_ranking') + row.pop('contest_id', None) + row.pop('global_ranking', None) r = result.setdefault(member, OrderedDict()) @@ -230,14 +237,14 @@ def fetch_ranking(domain, region, n_page=None): stats = get_item(statistics, member, {}) problems = r.setdefault('problems', get_copy_item(stats, 'problems', {})) submitted_keys = set() - for i, (k, s) in enumerate(submissions.items(), start=1): + for k, s in submissions.items(): short = problems_info[k]['short'] submitted_keys.add(short) p = problems.setdefault(short, {}) time = datetime.fromtimestamp(s['date']) - start_time p['time_in_seconds'] = time.total_seconds() p['time'] = self.to_time(time) - if s['status'] == 10: + if 'status' not in s or s['status'] == 10: solved += 1 p['result'] = '+' + str(s['fail_count'] or '') else: @@ -247,7 +254,7 @@ def fetch_ranking(domain, region, n_page=None): if 'submission_id' in s: p['submission_id'] = s['submission_id'] p['external_solution'] = True - p['data_region'] = s['data_region'] + p['data_region'] = data_region skip = skip or p['submission_id'] in solutions_ids solutions_ids.add(p['submission_id']) @@ -275,10 +282,17 @@ def fetch_ranking(domain, region, n_page=None): r['place'] = rank r['info'] = {'profile_url': {'_domain': data_domain, '_handle': handle}} + if 'avatar_url' in row: + r['info']['userAvatar'] = row.pop('avatar_url') + + url = f"{standings_url.rstrip('/')}/{data['_page']}" + url = re.sub('[.][^./]+(?=/)', domain, url) + r['url'] = url country = None for field in 'country_code', 'country_name': - country = country or row.pop(field, None) + country_value = row.pop(field, None) + country = country or country_value if country: r['country'] = country @@ -296,8 +310,6 @@ def fetch_ranking(domain, region, n_page=None): if row.get('badge') or not row.get('user_badge'): row.pop('user_badge', None) - r.update(row) - for k in set(row): hidden_fields.add(k) if statistics and member in statistics: @@ -311,8 +323,9 @@ def fetch_ranking(domain, region, n_page=None): if n_added == 0 and not users: stop_fetch_standings = True - fetch_ranking(domain='.com', region='global') - fetch_ranking(domain='.cn', region='local') + fetch_ranking(domain='.com', region='global_v2') + if users is None or users: + fetch_ranking(domain='.cn', region='local_v2') if len(parsed_domains) > 1: def get_key(row): diff --git a/src/ranking/management/modules/open_kattis.py b/src/ranking/management/modules/open_kattis.py index 82808eb6..cc59c824 100644 --- a/src/ranking/management/modules/open_kattis.py +++ b/src/ranking/management/modules/open_kattis.py @@ -12,7 +12,7 @@ from tqdm import tqdm from clist.models import Contest -from clist.templatetags.extras import as_number +from clist.templatetags.extras import as_number, get_item, slug from ranking.management.modules.common import REQ, BaseModule, FailOnGetResponse, parsed_table @@ -22,6 +22,8 @@ def get_standings(self, users=None, statistics=None, **kwargs): url = self.url.split('?')[0].rstrip('/') standings_url = url + '/standings' problems_url = url + '/problems' + subdomain = urlparse(standings_url).netloc.split('.')[0] + has_subdomain = subdomain not in ('open', 'kattis') result = {} problems_info = OrderedDict() @@ -39,116 +41,146 @@ def get_standings(self, users=None, statistics=None, **kwargs): table = parsed_table.ParsedTable(html_table) for r in table: - short = r[''].value + short = r.pop('').value problem_info = problems_info.setdefault(short, {'short': short}) - problem_info['name'] = r['Name'].value - url = r['Name'].column.node.xpath('.//a/@href') - if url: - url = urljoin(problems_url, url[0]) - problem_info['url'] = url - code = url.rstrip('/').rsplit('/', 1)[-1] - problem_info['code'] = code + if 'Name' in r: + name = r.pop('Name') + problem_info['name'] = name.value + url = name.column.node.xpath('.//a/@href') + if url: + url = urljoin(problems_url, url[0]) + problem_info['url'] = url + code = url.rstrip('/').rsplit('/', 1)[-1] + if has_subdomain: + code = f'{subdomain}:{code}' + problem_info['code'] = code + if r: + more_fields = problem_info.setdefault('_more_fields', {}) + for k, v in r.items(): + more_fields[slug(k, sep='_')] = as_number(v.value) page = REQ.get(standings_url) - - regex = '<table[^>]*class="[^"]*standings-table[^"]*"[^>]*>.*?</table>' - entry = re.search(regex, page, re.DOTALL) - html_table = entry.group(0) - table = parsed_table.ParsedTable(html_table) + standings_urls = [standings_url] + options = re.findall(r'value="(?P<option>\?filter=[0-9]+)"', page) + for option in options: + standings_urls.append(urljoin(standings_url, option)) rows = [] standings_kind = Contest.STANDINGS_KINDS['icpc'] - for r in table: - row = {} - problems = row.setdefault('problems', {}) - - if 'Rk' in r: - row['place'] = r.pop('Rk').value - - team = r.pop('Team') - if isinstance(team, list): - if 'place' not in row: - v, *team = team - row['place'] = v.value - team, *more = team - for addition in more: - for img in addition.column.node.xpath('.//img'): - alt = img.attrib.get('alt', '') - if not alt: - continue - src = img.attrib.get('src', '') - if '/countries/' in src and 'country' not in row: - row['country'] = alt - elif '/universities/' in src and 'university' not in row: - row['university'] = alt - if not team.value: - continue - - row['name'] = team.value - - if 'Slv.' in r: - row['solving'] = as_number(r.pop('Slv.').value, force=True) - elif 'Score' in r: - row['solving'] = as_number(r.pop('Score').value, force=True) - - if 'Time' in r: - row['penalty'] = int(r.pop('Time').value) - - for k, v in r.items(): - k, *other = k.split() - if len(k) == 1: - full_score = None - if other and (match := re.match(r'\((\d+)\)', other[0])): - full_score = int(match.group(1)) - problems_info[k].setdefault('full_score', full_score) - standings_kind = Contest.STANDINGS_KINDS['scoring'] - - if not v.value: + seen_urls = set() + with PoolExecutor(max_workers=10) as executor: + for page in executor.map(REQ.get, standings_urls): + regex = '<table[^>]*class="[^"]*standings-table[^"]*"[^>]*>.*?</table>' + entry = re.search(regex, page, re.DOTALL) + html_table = entry.group(0) + table = parsed_table.ParsedTable(html_table) + + for r in table: + row = {} + problems = row.setdefault('problems', {}) + + if 'Rk' in r: + row['place'] = r.pop('Rk').value + + team = r.pop('Team') + if isinstance(team, list): + if 'place' not in row: + v, *team = team + row['place'] = v.value + team, *more = team + for addition in more: + for img in addition.column.node.xpath('.//img'): + alt = img.attrib.get('alt', '') + if not alt: + continue + src = img.attrib.get('src', '') + if '/countries/' in src and 'country' not in row: + row['country'] = alt + elif '/universities/' in src and 'university' not in row: + row['university'] = alt + if not team.value: continue - p = problems.setdefault(k, {}) - - score, *values = v.value.split() - if '+' in score: - score = sum(map(int, score.split('+'))) - else: - score = as_number(score) - classes = v.column.node.xpath('@class')[0].split() - - pending = 'pending' in classes - first = 'solvedfirst' in classes or bool(v.column.node.xpath('.//i[contains(@class,"cell-first")]')) - solved = first or 'solved' in classes - - if not full_score: - if solved: - p['result'] = '+' if score == 1 else f'+{score - 1}' - p['time'] = self.to_time(int(values[0]), 2) - elif pending: - p['result'] = '?' if score == 1 else f'?{score - 1}' - else: - p['result'] = f'-{score}' - else: - p['result'] = score - p['partial'] = not solved and full_score > score - - if first: - p['first_ac'] = True - - if not problems: - continue + row['name'] = team.value + + if 'Slv.' in r: + row['solving'] = as_number(r.pop('Slv.').value, force=True) + elif 'Score' in r: + row['solving'] = as_number(r.pop('Score').value, force=True) + + if 'Time' in r: + row['penalty'] = int(r.pop('Time').value) + + for k, v in r.items(): + k, *other = k.split() + if len(k) == 1: + full_score = None + if other and (match := re.match(r'\((\d+)\)', other[0])): + full_score = int(match.group(1)) + problems_info[k].setdefault('full_score', full_score) + standings_kind = Contest.STANDINGS_KINDS['scoring'] + + if not v.value: + continue + + p = problems.setdefault(k, {}) + + score, *values = v.value.split() + if '+' in score: + score = sum(map(int, score.split('+'))) + else: + score = as_number(score) + classes = v.column.node.xpath('@class')[0].split() + + pending = 'pending' in classes + first = 'solvedfirst' in classes + first = first or bool(v.column.node.xpath('.//i[contains(@class,"cell-first")]')) + solved = first or 'solved' in classes + if options: + first = False + + if not full_score: + if solved: + p['result'] = '+' if score == 1 else f'+{score - 1}' + p['time'] = self.to_time(int(values[0]), 2) + elif pending: + p['result'] = '?' if score == 1 else f'?{score - 1}' + else: + p['result'] = f'-{score}' + else: + p['result'] = score + p['partial'] = not solved and full_score > score + + if first: + p['first_ac'] = True + + if not problems: + continue - urls = team.column.node.xpath('.//a/@href') - assert len(urls) == 1 - url = urls[0] - assert url.startswith('/contests/') - row['_account_url'] = urljoin(standings_url, url) - rows.append(row) + urls = team.column.node.xpath('.//a/@href') + assert len(urls) == 1 + url = urls[0] + assert url.startswith('/contests/') + url = urljoin(standings_url, url) + if url in seen_urls: + continue + seen_urls.add(url) + row['_account_url'] = url + rows.append(row) + if options: + sorted_rows = sorted(rows, key=lambda x: (-x.get('solving', 0), x.get('penalty', 0))) + last_rank = None + last_score = None + for rank, row in enumerate(sorted_rows, start=1): + score = (row.get('solving', 0), row.get('penalty', 0)) + if last_score != score: + last_rank = rank + last_score = score + row['place'] = last_rank with PoolExecutor(max_workers=10) as executor: - - subdomain = urlparse(standings_url).netloc.split('.')[0] - has_subdomain = subdomain not in ('open', 'kattis') + split_team = get_item(self.resource.info, 'statistics.split_team', default=True) def fetch_members(row): page = REQ.get(row['_account_url']) @@ -178,12 +210,23 @@ def fetch_members(row): if real_members: members = real_members - for member in members: - row['member'] = member['username'] - account_info = row.setdefault('info', {}) - account_info['name'] = member['name'] - account_info['profile_url'] = member.get('profile_url') - result[row['member']] = deepcopy(row) + if split_team: + for member in members: + row['member'] = member['username'] + account_info = row.setdefault('info', {}) + account_info['name'] = member['name'] + account_info['profile_url'] = member.get('profile_url') + result[row['member']] = deepcopy(row) + else: + for member in row['_members']: + member.pop('account', None) + member = f'team-{row["team_id"]}' + if has_subdomain: + member = f'{subdomain}:{member}' + row['member'] = member + row.pop('team_id') + row.pop('_account_url') + result[member] = deepcopy(row) standings = { 'result': result, diff --git a/src/ranking/management/modules/topcoder.py b/src/ranking/management/modules/topcoder.py index f7f4eb9f..9490dbf6 100644 --- a/src/ranking/management/modules/topcoder.py +++ b/src/ranking/management/modules/topcoder.py @@ -22,6 +22,20 @@ from utils.requester import FailOnGetResponse +def parse_xml(page, exc=None): + try: + root = ET.fromstring(page) + except ET.ParseError as e: + if exc is not None: + raise exc(f'Failed to parse xml: {e}') + raise e + for child in root: + data = {} + for field in child: + data[field.tag] = field.text + yield data + + class Statistic(BaseModule): LEGACY_PROXY_PATH = 'logs/legacy/topcoder.proxy' @@ -104,6 +118,7 @@ def get_standings(self, users=None, statistics=None, **kwargs): filepath_proxies='sharedfiles/resource/topcoder/proxies', connect=lambda req: req.get('https://www.topcoder.com/', n_attempts=1), attributes=dict(n_attempts=5), + inplace=False, ) if not self.standings_url and datetime.now() - start_time < timedelta(days=30): @@ -161,11 +176,7 @@ def process_match(date, title, url): url = 'https://www.topcoder.com/tc?module=BasicData&c=dd_round_list' page = req.get(url) - root = ET.fromstring(page) - for child in root: - data = {} - for field in child: - data[field.tag] = field.text + for data in parse_xml(page, exc=ExceptionParseStandings): date = dateutil.parser.parse(data['date']) url = 'https://www.topcoder.com/stat?c=round_overview&er=5&rd=' + data['round_id'] process_match(date, data['full_name'], url) @@ -263,11 +274,7 @@ def process_match(date, title, url): url = f'https://www.topcoder.com/tc?module=BasicData&c=dd_round_results&rd={rd}' try: dd_round_results_page = req.get(url) - root = ET.fromstring(dd_round_results_page) - for child in root: - data = {} - for field in child: - data[field.tag] = field.text + for data in parse_xml(dd_round_results_page, exc=ExceptionParseStandings): handle = data.pop('handle') dd_round_results[handle] = self._dict_as_number(data) except FailOnGetResponse: @@ -589,14 +596,6 @@ def fetch_info(url): @staticmethod def get_users_infos(users, resource=None, accounts=None, pbar=None): - def parse_xml(page): - root = ET.fromstring(page) - for child in root: - data = {} - for field in child: - data[field.tag] = field.text - yield data - active_algorithm_list_url = 'https://www.topcoder.com/tc?module=BasicData&c=dd_active_algorithm_list' with REQ.with_proxy( time_limit=10, @@ -607,7 +606,7 @@ def parse_xml(page): ) as req: page = req.proxer.get_connect_ret() dd_active_algorithm = {} - for data in parse_xml(page): + for data in parse_xml(page, exc=ExceptionParseAccounts): dd_active_algorithm[data.pop('handle')] = data def fetch_profile(user): @@ -652,7 +651,7 @@ def fetch_profile(user): page = req.get(url) max_rating_order = -1 ret['rating'], ret['volatility'], n_rating = None, None, 0 - for data in parse_xml(page): + for data in parse_xml(page, exc=ExceptionParseAccounts): n_rating += 1 rating_order = as_number(data['rating_order']) if rating_order > max_rating_order: diff --git a/src/ranking/management/modules/yandex.py b/src/ranking/management/modules/yandex.py index 9b641a24..33838b6f 100644 --- a/src/ranking/management/modules/yandex.py +++ b/src/ranking/management/modules/yandex.py @@ -12,8 +12,8 @@ from clist.templatetags.extras import get_item from my_oauth.models import Service from ranking.management.modules.common import LOG, REQ, BaseModule, FailOnGetResponse, parsed_table -from ranking.management.modules.excepts import ExceptionParseStandings from ranking.management.modules.common.locator import Locator +from ranking.management.modules.excepts import ExceptionParseStandings class Statistic(BaseModule): @@ -50,7 +50,7 @@ def get_submission_infos(self, statistics, names): if not coder_pk: return submission_infos - ouath_service = Service.objects.get(name='competitive-hustle') + ouath_service = Service.objects.get(name='yandex-contest') oauth_token = ouath_service.token_set.filter(coder__pk=coder_pk).first() if not oauth_token: return submission_infos diff --git a/src/ranking/migrations/0133_subsriptions_and_parsestatistics.py b/src/ranking/migrations/0133_subsriptions_and_parsestatistics.py new file mode 100644 index 00000000..9e20c00c --- /dev/null +++ b/src/ranking/migrations/0133_subsriptions_and_parsestatistics.py @@ -0,0 +1,45 @@ +# Generated by Django 5.1 on 2024-11-17 15:29 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('ranking', '0133_remove_account_is_subscribed_account_n_subsriptions'), ('ranking', '0134_rename_n_subsriptions_account_n_subscriptions'), ('ranking', '0135_alter_account_n_subscriptions'), ('ranking', '0136_rename_n_subscriptions_account_n_subscribers'), ('ranking', '0137_parsestatistics'), ('ranking', '0138_rename_parsestatistics_parsestatistic'), ('ranking', '0139_rename_parsestatistic_parsestatistics_and_more'), ('ranking', '0140_parsestatistics_parse_time')] + + dependencies = [ + ('clist', '0167_rename_inherit_medals_to_related_resource_has_inherit_medals_to_related'), + ('ranking', '0132_account_rating_prediction'), + ] + + operations = [ + migrations.RemoveField( + model_name='account', + name='is_subscribed', + ), + migrations.AddField( + model_name='account', + name='n_subscribers', + field=models.IntegerField(blank=True, db_index=True, default=0), + ), + migrations.CreateModel( + name='ParseStatistics', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('modified', models.DateTimeField(auto_now=True, db_index=True)), + ('delay', models.DurationField(blank=True, null=True)), + ('enable', models.BooleanField(blank=True, default=True)), + ('without_set_coder_problems', models.BooleanField(blank=True, default=True)), + ('without_stage', models.BooleanField(blank=True, default=True)), + ('without_subscriptions', models.BooleanField(blank=True, default=False)), + ('contest', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='clist.contest')), + ('parse_time', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'abstract': False, + 'verbose_name_plural': 'ParseStatistics', + }, + ), + ] diff --git a/src/ranking/models.py b/src/ranking/models.py index 744f7f0c..f7e74e0c 100644 --- a/src/ranking/models.py +++ b/src/ranking/models.py @@ -1,20 +1,15 @@ -import ast -import collections import hashlib import os import re -from copy import deepcopy -from pydoc import locate from urllib.parse import quote, urljoin import magic import requests -import tqdm from django.conf import settings from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.core.management import call_command -from django.db import models, transaction +from django.db import models from django.db.models import F, OuterRef, Q, Sum from django.db.models.functions import Coalesce, Upper from django.db.models.signals import m2m_changed, post_delete, post_save, pre_save @@ -23,11 +18,10 @@ from django.utils import timezone from django.utils.crypto import get_random_string from django_countries.fields import CountryField -from django_print_sql import print_sql from sql_util.utils import Exists, SubqueryCount, SubquerySum from clist.models import Contest, Resource -from clist.templatetags.extras import add_prefix_to_problem_short, get_problem_short, has_season, slug +from clist.templatetags.extras import get_item, has_season from pyclist.indexes import ExpressionIndex, GistIndexTrgrmOps from pyclist.models import BaseManager, BaseModel from true_coders.models import Coder, Party @@ -44,6 +38,7 @@ class Account(BaseModel): url = models.CharField(max_length=4096, null=True, blank=True) n_contests = models.IntegerField(default=0, db_index=True) n_writers = models.IntegerField(default=0, db_index=True) + n_subscribers = models.IntegerField(default=0, db_index=True, blank=True) last_activity = models.DateTimeField(default=None, null=True, blank=True, db_index=True) last_submission = models.DateTimeField(default=None, null=True, blank=True, db_index=True) last_rating_activity = models.DateTimeField(default=None, null=True, blank=True, db_index=True) @@ -55,7 +50,6 @@ class Account(BaseModel): duplicate = models.ForeignKey('Account', null=True, blank=True, on_delete=models.CASCADE) global_rating = models.IntegerField(null=True, blank=True, default=None, db_index=True) need_verification = models.BooleanField(default=False) - is_subscribed = models.BooleanField(null=True, blank=True, default=None, db_index=True) deleted = models.BooleanField(null=True, blank=True, default=None, db_index=True) try_renaming_check_time = models.DateTimeField(null=True, blank=True, default=None) try_fill_missed_ranks_time = models.DateTimeField(null=True, blank=True, default=None) @@ -143,12 +137,22 @@ def update_last_rating_activity(self, statistic, contest=None, resource=None): def display(self, with_resource=None): if not with_resource and self.name and has_season(self.key, self.name): ret = self.name + elif not self.name or self.name == self.key: + ret = self.key else: ret = f'{self.key}, {self.name}' if with_resource: - ret += f' ({self.resource.host})' + ret += f', {self.resource.host}' return ret + def short_display(self, resource=None, name=None): + name = name or self.name + resource = resource or self.resource + name_instead_key = get_item(resource, 'info.standings.name_instead_key') + name_instead_key = get_item(self, 'info._name_instead_key', default=name_instead_key) + name_instead_key = name and (name_instead_key or has_season(self.key, name)) + return name if name_instead_key else self.key + def save(self, *args, **kwargs): update_fields = kwargs.get('update_fields') prev_rating = self.rating @@ -373,9 +377,8 @@ def update_account_writer(signal, instance, action, reverse, pk_set, **kwargs): return if reverse: - instance.n_writers += delta - instance.save() - else: + Account.objects.filter(pk=instance.pk).update(n_writers=F('n_writers') + delta) + elif pk_set: Account.objects.filter(pk__in=pk_set).update(n_writers=F('n_writers') + delta) @@ -387,7 +390,7 @@ def update_coder_n_accounts_and_n_contests(signal, instance, action, reverse, pk if reverse: instance.n_accounts = instance.account_set.count() instance.n_contests = instance.account_set.aggregate(total=Sum('n_contests'))['total'] or 0 - instance.save() + instance.save(update_fields=['n_accounts', 'n_contests']) resources = Resource.objects.annotate(has_account=Exists('account', filter=Q(pk__in=pk_set))) resources = resources.filter(has_account=True) @@ -403,7 +406,7 @@ def update_coder_n_accounts_and_n_contests(signal, instance, action, reverse, pk resources = [instance.resource.host] if coders and resources: - call_command('fill_coder_problems', coders=coders, resources=resources) + call_command('set_coder_problems', coders=coders, resources=resources) class AccountVerification(BaseModel): @@ -477,7 +480,7 @@ def is_special_addition_field(field): return field in settings.ADDITION_HIDE_FIELDS_ def __str__(self): - return f'Statistics#{self.id} {self.account_id} on {self.contest_id}' + return f'Statistics#{self.id} account#{self.account_id} on contest#{self.contest_id}' def get_old_rating(self, use_rating_prediction=True): rating_datas = [self.addition] @@ -497,6 +500,23 @@ def is_rated(self): return False return 'new_rating' in self.addition or 'rating_change' in self.addition + @property + def account_name(self): + resource = ( + self.fetched_field('contest__resource') or + self.fetched_field('account__resource') or + self.contest.resource + ) + return self.account.short_display(resource=resource, name=self.addition.get('name')) + + @classmethod + def top_n_filter(cls, n): + return Q(place_as_int__lte=n) + + @classmethod + def first_ac_filter(cls): + return Q(addition__problems__icontains='"first_ac": true') + class Meta: verbose_name_plural = 'Statistics' unique_together = ('account', 'contest') @@ -569,640 +589,6 @@ class Stage(BaseModel): def __str__(self): return 'Stage#%d %s' % (self.pk, self.contest) - def update(self): - eps = 1e-9 - stage = self.contest - timezone_now = timezone.now() - - filter_params = dict(self.filter_params) - spec_filter_params = dict() - for field in ('info__fields_types__new_rating__isnull',): - if field in filter_params: - spec_filter_params[field] = filter_params.pop(field) - is_over = self.contest.end_time < timezone_now - - contests = Contest.objects.filter( - resource=self.contest.resource, - start_time__gte=self.contest.start_time, - end_time__lte=self.contest.end_time, - **filter_params, - ).exclude(pk=self.contest.pk) - - if spec_filter_params: - contests = contests.filter(Q(**spec_filter_params) | Q(end_time__gt=timezone_now)) - - contests = contests.order_by('start_time') - contests = contests.prefetch_related('writers') - self.contests.set(contests) - - parsed_statistic = self.score_params.get('parse_statistic') - if parsed_statistic: - call_command('parse_statistic', - contest_id=stage.pk, - without_fill_coder_problems=True, - ignore_stage=True) - stage.refresh_from_db() - - placing = self.score_params.get('place') - n_best = self.score_params.get('n_best') - fields = self.score_params.get('fields', []) - scoring = self.score_params.get('scoring', {}) - detail_problems = self.score_params.get('detail_problems') - order_by = self.score_params['order_by'] - advances = self.score_params.get('advances', {}) - results = collections.defaultdict(collections.OrderedDict) - - mapping_account_by_coder = {} - fixed_fields = [] - hidden_fields = [] - - problems_infos = collections.OrderedDict() - divisions_order = [] - for idx, contest in enumerate(tqdm.tqdm(contests, desc=f'getting contests for stage {stage}'), start=1): - info = { - 'code': str(contest.pk), - 'name': contest.title, - 'url': reverse( - 'ranking:standings', - kwargs={'title_slug': slug(contest.title), 'contest_id': str(contest.pk)} - ), - } - - if contest.start_time > timezone_now: - info['subtext'] = {'text': 'upcoming', 'title': str(contest.start_time)} - - for division in contest.info.get('divisions_order', []): - if division not in divisions_order: - divisions_order.append(division) - - if self.score_params.get('regex_problem_name'): - match = re.search(self.score_params.get('regex_problem_name'), contest.title) - if match: - info['short'] = match.group(1) - if self.score_params.get('abbreviation_problem_name'): - info['short'] = ''.join(re.findall(r'(\b[A-Z]|[0-9])', info.get('short', contest.title))) - - problems = contest.info.get('problems', []) - if not detail_problems: - full_score = None - if placing: - if 'division' in placing: - full_score = max([max(p.values()) for p in placing['division'].values()]) - else: - full_score = max(placing.values()) - elif 'division' in problems: - full_scores = [] - for ps in problems['division'].values(): - full = 0 - for problem in ps: - full += problem.get('full_score', 1) - full_scores.append(full) - info['full_score'] = max(full_scores) - elif self.score_params.get('default_problem_full_score'): - full_score = self.score_params['default_problem_full_score'] - else: - full_score = 0 - for problem in problems: - full_score += problem.get('full_score', 1) - if full_score is not None: - info['full_score'] = full_score - problems_infos[str(contest.pk)] = info - else: - for problem in problems: - problem = dict(problem) - add_prefix_to_problem_short(problem, f'{idx}.') - problem['group'] = info.get('short', info['name']) - problem['url'] = info['url'] - problems_infos[get_problem_short(problem)] = problem - - exclude_advances = {} - if advances and advances.get('exclude_stages'): - qs = Statistics.objects \ - .filter(contest__stage__in=advances['exclude_stages'], addition___advance__isnull=False) \ - .values('account__key', 'addition___advance', 'contest__title') \ - .order_by('contest__end_time', 'contest__id') - for r in qs: - d = r['addition___advance'] - if 'contest' not in d: - d['contest'] = r['contest__title'] - exclude_advances[r['account__key']] = d - - statistics = Statistics.objects \ - .select_related('account', 'account__duplicate') \ - .prefetch_related('account__coders') - filter_statistics = self.score_params.get('filter_statistics') - if filter_statistics: - statistics = statistics.filter(**filter_statistics) - exclude_statistics = self.score_params.get('exclude_statistics') - if exclude_statistics: - statistics = statistics.exclude(**exclude_statistics) - - def get_placing(placing, stat): - return placing['division'][stat.addition['division']] if 'division' in placing else placing - - account_keys = dict() - total = statistics.filter(contest__in=contests).count() - with tqdm.tqdm(total=total, desc=f'getting statistics for stage {stage}') as pbar, print_sql(count_only=True): - for idx, contest in enumerate(contests, start=1): - skip_problem_stat = '_skip_for_problem_stat' in contest.info.get('fields', []) - contest_unrated = contest.info.get('unrated') - - if not detail_problems: - problem_info_key = str(contest.pk) - problem_short = get_problem_short(problems_infos[problem_info_key]) - pbar.set_postfix(contest=contest) - stats = statistics.filter(contest_id=contest.pk) - - if placing: - placing_scores = deepcopy(placing) - n_rows = 0 - for s in stats: - n_rows += 1 - placing_ = get_placing(placing_scores, s) - key = str(s.place_as_int) - if key in placing_: - placing_.setdefault('scores', {}) - placing_['scores'][key] = placing_.pop(key) - scores = [] - for place in reversed(range(1, n_rows + 1)): - placing_ = get_placing(placing_scores, s) - key = str(place) - if key in placing_: - scores.append(placing_.pop(key)) - else: - if scores: - placing_['scores'][key] += sum(scores) - placing_['scores'][key] /= len(scores) + 1 - scores = [] - - max_solving = 0 - n_effective = 0 - for stat in stats: - max_solving = max(max_solving, stat.solving) - n_effective += stat.solving > eps - - for s in stats: - if not detail_problems and not skip_problem_stat: - problems_infos[problem_info_key].setdefault('n_total', 0) - problems_infos[problem_info_key]['n_total'] += 1 - - score = None - if s.solving < eps: - score = 0 - if placing: - placing_ = get_placing(placing_scores, s) - score = placing_.get('zero', 0) - else: - if placing: - placing_ = get_placing(placing_scores, s) - score = placing_['scores'].get(str(s.place_as_int), placing_.get('default')) - if score is None: - continue - if scoring: - if score is None: - score = 0 - if scoring['name'] == 'general': - if s.place_as_int is None: - continue - solving_factor = s.solving / max_solving - rank_factor = (n_effective - s.place_as_int + 1) / n_effective - score += scoring['factor'] * solving_factor * rank_factor - elif scoring['name'] == 'field': - score += s.addition.get(scoring['field'], 0) - else: - raise NotImplementedError(f'scoring {scoring["name"]} is not implemented') - if score is None: - score = s.solving - - if not detail_problems and not skip_problem_stat: - problems_infos[problem_info_key].setdefault('n_teams', 0) - problems_infos[problem_info_key]['n_teams'] += 1 - if score: - problems_infos[problem_info_key].setdefault('n_accepted', 0) - problems_infos[problem_info_key]['n_accepted'] += 1 - - account = s.account - if account.duplicate is not None: - account = account.duplicate - - coders = account.coders.all() - has_mapping_account_by_coder = False - if len(coders) == 1: - coder = coders[0] - if coder not in mapping_account_by_coder: - mapping_account_by_coder[coder] = account - else: - account = mapping_account_by_coder[coder] - has_mapping_account_by_coder = True - - row = results[account] - row['member'] = account - account_keys[account.key] = account - - problems = row.setdefault('problems', {}) - if detail_problems: - for key, problem in s.addition.get('problems', {}).items(): - p = problems.setdefault(f'{idx}.' + key, {}) - if contest_unrated: - p = p.setdefault('upsolving', {}) - p.update(problem) - else: - problem = problems.setdefault(problem_short, {}) - if contest_unrated: - problem = problem.setdefault('upsolving', {}) - problem['result'] = score - url = s.addition.get('url') - if not url: - url = reverse('ranking:standings_by_id', kwargs={'contest_id': str(contest.pk)}) - url += f'?find_me={s.pk}' - problem['url'] = url - if contest_unrated: - score = 0 - - if n_best and not detail_problems: - row.setdefault('scores', []).append((score, problem)) - else: - row['score'] = row.get('score', 0) + score - - field_values = {} - for field in fields: - if field.get('skip_on_mapping_account_by_coder') and has_mapping_account_by_coder: - continue - if 'type' in field: - continue - inp = field['field'] - out = field.get('out', inp) - if field.get('first') and out in row or (inp not in s.addition and not hasattr(s, inp)): - continue - val = s.addition[inp] if inp in s.addition else getattr(s, inp) - if not field.get('safe') and isinstance(val, str): - val = ast.literal_eval(val) - if 'cast' in field: - val = locate(field['cast'])(val) - field_values[out] = val - if field.get('skip'): - continue - if field.get('accumulate'): - val = round(val + ast.literal_eval(str(row.get(out, 0))), 2) - if field.get('aggregate') == 'avg': - out_n = f'_{out}_n' - out_s = f'_{out}_s' - row[out_n] = row.get(out_n, 0) + 1 - row[out_s] = row.get(out_s, 0) + val - val = round(row[out_s] / row[out_n], 2) - if field.get('aggregate') == 'max' and out in row: - val = max(val, row[out]) - row[out] = val - - if 'solved' in s.addition and isinstance(s.addition['solved'], dict): - solved = row.setdefault('solved', {}) - for k, v in s.addition['solved'].items(): - solved[k] = solved.get(k, 0) + v - - if 'status' in self.score_params: - field = self.score_params['status'] - val = field_values.get(field, row.get(field)) - if val is None: - val = getattr(s, field) - if val is not None: - problem['status'] = val - else: - for field in order_by: - field = field.lstrip('-') - if field in ['score', 'rating', 'penalty']: - continue - status = field_values.get(field, row.get(field)) - if status is None: - continue - problem['status'] = status - break - pbar.update() - - for writer in contest.writers.all(): - account_keys[writer.key] = writer - - total = sum([len(contest.info.get('writers', [])) for contest in contests]) - with tqdm.tqdm(total=total, desc=f'getting writers for stage {stage}') as pbar, print_sql(count_only=True): - writers = set() - for contest in contests: - contest_writers = contest.info.get('writers', []) - if not contest_writers or detail_problems: - continue - problem_info_key = str(contest.pk) - problem_short = get_problem_short(problems_infos[problem_info_key]) - for writer in contest_writers: - if writer in account_keys: - account = account_keys[writer] - else: - try: - account = Account.objects.get(resource_id=contest.resource_id, key__iexact=writer) - except Account.DoesNotExist: - account = None - - pbar.update() - if not account: - continue - writers.add(account) - - row = results[account] - row['member'] = account - row.setdefault('score', 0) - if n_best: - row.setdefault('scores', []) - row.setdefault('writer', 0) - - row['writer'] += 1 - - problems = row.setdefault('problems', {}) - problem = problems.setdefault(problem_short, {}) - problem['status'] = 'W' - - if self.score_params.get('writers_proportionally_score'): - n_contests = sum(contest.start_time < timezone_now for contest in contests) - for account in writers: - row = results[account] - if n_contests == row['writer'] or 'score' not in row: - continue - row['score'] = row['score'] / (n_contests - row['writer']) * n_contests - if self.score_params.get('exponential_score_decay'): - for r in results.values(): - scores = [problem.get('result', 0) for problem in r.get('problems', {}).values()] - scores.sort(reverse=True) - k = self.score_params['exponential_score_decay'] - score = k * sum((1 - k) ** i * score for i, score in enumerate(scores)) - r['score'] = score - - for field in fields: - t = field.get('type') - if t is None: - continue - if t == 'points_for_common_problems': - group = field['group'] - inp = field['field'] - out = field.get('out', inp) - - excluding = bool(exclude_advances and field.get('exclude_advances')) - - groups = collections.defaultdict(list) - for row in results.values(): - key = row[group] - groups[key].append(row) - - advancement_position = 1 - for key, rows in sorted(groups.items(), key=lambda kv: kv[0], reverse=True): - common_problems = None - for row in rows: - handle = row['member'].key - if excluding and handle in exclude_advances: - exclude_advance = exclude_advances[handle] - for advance in advances.get('options', []): - if advance['next'] == exclude_advance['next']: - exclude_advance['skip'] = True - break - if advancement_position in advance['places']: - break - if exclude_advance.get('skip'): - continue - - problems = {k for k, p in row['problems'].items() if p.get('status') != 'W'} - common_problems = problems if common_problems is None else (problems & common_problems) - if common_problems is None: - for row in rows: - problems = {k for k, p in row['problems'].items() if p.get('status') != 'W'} - common_problems = problems if common_problems is None else (problems & common_problems) - - for row in rows: - handle = row['member'].key - if excluding and not exclude_advances.get(handle, {}).get('skip', False): - advancement_position += 1 - value = 0 - for k in common_problems: - value += float(row['problems'].get(k, {}).get(inp, 0)) - for k, v in row['problems'].items(): - if k not in common_problems and v.get('status') != 'W': - v['status_tag'] = 'strike' - row[out] = round(value, 2) - elif t == 'region_by_country': - out = field['out'] - - mapping_regions = dict() - for regional_event in field['data']: - for region in regional_event['regions']: - mapping_regions[region['code']] = {'regional_event': regional_event, 'region': region} - - for row in results.values(): - country = row['member'].country - if not country: - continue - code = country.code - if code not in mapping_regions: - continue - row[out] = mapping_regions[code]['regional_event']['name'] - elif t == 'n_medal_problems': - for row in results.values(): - for problem in row['problems'].values(): - medal = problem.get('medal') - if medal: - k = f'n_{medal}_problems' - row.setdefault(k, 0) - row[k] += 1 - if k not in hidden_fields: - hidden_fields.append(k) - for field in settings.STANDINGS_FIELDS_: - if field in hidden_fields: - fixed_fields.append(field) - hidden_fields.remove(field) - else: - raise ValueError(f'Unknown field type = {t}') - - hidden_fields += [field.get('out', field.get('inp')) for field in fields if field.get('hidden')] - - results = list(results.values()) - if n_best: - for row in results: - scores = row.pop('scores') - for index, (score, problem) in enumerate(sorted(scores, key=lambda s: s[0], reverse=True)): - if index < n_best: - row['score'] = row.get('score', 0) + score - else: - problem['status'] = problem.pop('result') - - filtered_results = [] - filter_zero_points = self.score_params.get('filter_zero_points', True) - for r in results: - if r['score'] > eps or not filter_zero_points or r.get('writer'): - filtered_results.append(r) - continue - if detail_problems: - continue - - problems = r.setdefault('problems', {}) - - for idx, contest in enumerate(contests, start=1): - skip_problem_stat = '_skip_for_problem_stat' in contest.info.get('fields', []) - if skip_problem_stat: - continue - - problem_info_key = str(contest.pk) - problem_short = get_problem_short(problems_infos[problem_info_key]) - - if problem_short in problems: - problems_infos[problem_info_key].setdefault('n_teams', 0) - problems_infos[problem_info_key]['n_teams'] -= 1 - results = filtered_results - - results = sorted( - results, - key=lambda r: tuple(r.get(k.lstrip('-'), 0) * (-1 if k.startswith('-') else 1) for k in order_by), - reverse=True, - ) - - additions = deepcopy(stage.info.get('additions', {})) - field_to_problem = self.score_params.get('field_to_problem') - with transaction.atomic(): - fields_set = set() - fields = list() - - pks = set() - placing_infos = {} - score_advance = None - place_advance = 0 - place_index = 0 - for row in tqdm.tqdm(results, desc=f'update statistics for stage {stage}'): - for field in [row.get('member'), row.get('name')]: - row.update(additions.pop(field, {})) - - division = row.get('division', 'none') - placing_info = placing_infos.setdefault(division, {}) - placing_info['index'] = placing_info.get('index', 0) + 1 - - curr_score = tuple(row.get(k.lstrip('-'), 0) for k in order_by) - if curr_score != placing_info.get('last_score'): - placing_info['last_score'] = curr_score - placing_info['place'] = placing_info['index'] - - if advances and ('divisions' not in advances or division in advances['divisions']): - tmp = score_advance, place_advance, place_index - - place_index += 1 - if curr_score != score_advance: - score_advance = curr_score - place_advance = place_index - - for advance in advances.get('options', []): - handle = row['member'].key - if handle in exclude_advances and advance['next'] == exclude_advances[handle]['next']: - advance = exclude_advances[handle] - if 'class' in advance and not advance['class'].startswith('text-'): - advance['class'] = f'text-{advance["class"]}' - row['_advance'] = advance - break - - if 'places' in advance and place_advance in advance['places']: - if not advances.get('inplace_fields_only'): - row['_advance'] = advance - if is_over: - for field in advance.get('inplace_fields', []): - row[field] = advance[field] - tmp = None - break - - if tmp is not None: - score_advance, place_advance, place_index = tmp - account = row.pop('member') - solving = row.pop('score') - - advanced = False - if row.get('_advance'): - adv = row['_advance'] - advanced = not adv.get('skip') and not adv.get('class', '').startswith('text-') - - defaults = { - 'place': str(placing_info['place']), - 'place_as_int': placing_info['place'], - 'solving': solving, - 'addition': row, - 'skip_in_stats': True, - 'advanced': advanced, - } - if parsed_statistic: - defaults['place'] = None - defaults['place_as_int'] = None - defaults['solving'] = 0 - stat = Statistics.objects.filter(account=account, contest=stage).first() - if not stat: - continue - for k, v in defaults['addition'].items(): - if k not in stat.addition or (v and not stat.addition.get(k)): - stat.addition[k] = v - stat.skip_in_stats = defaults['skip_in_stats'] - stat.advanced = defaults['advanced'] - stat.save(update_fields=['addition', 'skip_in_stats', 'advanced']) - else: - stat, created = Statistics.objects.update_or_create( - account=account, - contest=stage, - defaults=defaults, - ) - pks.add(stat.pk) - - for k in stat.addition.keys(): - if field_to_problem and re.search(field_to_problem['regex'], k): - continue - if k not in fields_set: - fields_set.add(k) - fields.append(k) - stage.info['problems'] = list(problems_infos.values()) - - if field_to_problem: - for stat in stage.statistics_set.all(): - problems = stat.addition.setdefault('problems', {}) - was_updated = False - for k, v in list(stat.addition.items()): - match = re.search(field_to_problem['regex'], k) - if match: - problem_short = field_to_problem['format'].format(**match.groupdict()) - if problem_short not in problems: - problems[problem_short] = {'result': v} - stat.addition.pop(k) - if 'fields_types' in stage.info: - stage.info['fields_types'].pop(k, None) - was_updated = True - if was_updated: - stat.save(update_fields=['addition']) - - if parsed_statistic: - for field in stage.info.setdefault('hidden_fields', []): - if field not in hidden_fields: - hidden_fields.append(field) - regex_hidden_fields = self.score_params.get('regex_hidden_fields') - if regex_hidden_fields: - for field in fields: - if field not in hidden_fields and re.search(regex_hidden_fields, field): - hidden_fields.append(field) - stage.info['fields'] = list(fields) - stage.info['hidden_fields'] = hidden_fields - - fields_types = self.score_params.get('fields_types', {}) - if fields_types: - stage.info.setdefault('fields_types', {}).update(fields_types) - - if not parsed_statistic: - stage.statistics_set.exclude(pk__in=pks).delete() - stage.n_statistics = len(results) - stage.parsed_time = timezone_now - - standings_info = self.score_params.get('info', {}) - standings_info['fixed_fields'] = fixed_fields + [(f.lstrip('-'), f.lstrip('-')) for f in order_by] - stage.info['standings'] = standings_info - - if divisions_order and self.score_params.get('divisions_ordering'): - stage.info['divisions_order'] = divisions_order - - if stage.is_rated is None: - stage.is_rated = False - stage.save() - class VirtualStart(BaseModel): coder = models.ForeignKey(Coder, on_delete=models.CASCADE, related_name='virtual_starts') @@ -1271,4 +657,25 @@ class Meta: unique_together = ('name', 'statistic') def __str__(self): - return f'AccountMatching#{self.pk} {self.name}, statistic#{self.statistic_id}' + return f'AccountMatching#{self.pk} {self.name} statistic#{self.statistic_id}' + + +class ParseStatistics(BaseModel): + contest = models.ForeignKey(Contest, on_delete=models.CASCADE) + delay = models.DurationField(null=True, blank=True) + parse_time = models.DateTimeField(null=True, blank=True) + enable = models.BooleanField(default=True, blank=True) + without_set_coder_problems = models.BooleanField(default=True, blank=True) + without_stage = models.BooleanField(default=True, blank=True) + without_subscriptions = models.BooleanField(default=False, blank=True) + + class Meta: + verbose_name_plural = 'ParseStatistics' + + @staticmethod + def relevant_contest(): + contests = Contest.objects.filter(parsestatistics__isnull=False, end_time__gt=timezone.now()) + return contests.order_by('end_time').first() + + def __str__(self): + return f'ParseStatistics#{self.pk} contest#{self.contest_id}' diff --git a/src/ranking/utils.py b/src/ranking/utils.py index dc8b128e..17b774d3 100644 --- a/src/ranking/utils.py +++ b/src/ranking/utils.py @@ -1,22 +1,31 @@ #!/usr/bin/env python3 +import ast +import collections import functools import json import operator +import re from collections import defaultdict +from copy import deepcopy from datetime import timedelta +from pydoc import locate import tqdm +from django.conf import settings +from django.core.management import call_command from django.db import transaction from django.db.models import Q +from django.urls import reverse from django.utils import timezone +from django_print_sql import print_sql from django_super_deduper.merge import MergedModelInstance -# from django.core.management import call_command from sql_util.utils import Exists from clist.models import Contest +from clist.templatetags.extras import add_prefix_to_problem_short, get_item, get_problem_short, slug from ranking.management.modules.common import LOG -from ranking.models import AccountRenaming, Statistics +from ranking.models import Account, AccountRenaming, Statistics from utils.logger import suppress_db_logging_context from utils.mathutils import max_with_none @@ -350,3 +359,655 @@ def create_upsolving_statistic(contest, account): stat.addition['_no_update_n_contests'] = True stat.save(update_fields=['skip_in_stats', 'addition']) return stat, created + + +def update_stage(self): + eps = 1e-9 + stage = self.contest + timezone_now = timezone.now() + + filter_params = dict(self.filter_params) + spec_filter_params = dict() + for field in ('info__fields_types__new_rating__isnull',): + if field in filter_params: + spec_filter_params[field] = filter_params.pop(field) + is_over = self.contest.end_time < timezone_now + + contests = Contest.objects.filter( + resource=self.contest.resource, + start_time__gte=self.contest.start_time, + end_time__lte=self.contest.end_time, + **filter_params, + ).exclude(pk=self.contest.pk) + + if spec_filter_params: + contests = contests.filter(Q(**spec_filter_params) | Q(end_time__gt=timezone_now)) + + contests = contests.order_by('start_time') + contests = contests.prefetch_related('writers') + self.contests.set(contests) + + parsed_statistic = self.score_params.get('parse_statistic') + if parsed_statistic: + call_command('parse_statistic', + contest_id=stage.pk, + without_set_coder_problems=True, + ignore_stage=True) + stage.refresh_from_db() + + placing = self.score_params.get('place') + n_best = self.score_params.get('n_best') + fields = self.score_params.get('fields', []) + scoring = self.score_params.get('scoring', {}) + detail_problems = self.score_params.get('detail_problems') + order_by = self.score_params['order_by'] + advances = self.score_params.get('advances', {}) + results = collections.defaultdict(collections.OrderedDict) + + mapping_account_by_coder = {} + fixed_fields = [] + hidden_fields = [] + + problems_infos = collections.OrderedDict() + divisions_order = [] + for idx, contest in enumerate(tqdm.tqdm(contests, desc=f'getting contests for stage {stage}'), start=1): + info = { + 'code': str(contest.pk), + 'name': contest.title, + 'url': reverse( + 'ranking:standings', + kwargs={'title_slug': slug(contest.title), 'contest_id': str(contest.pk)} + ), + } + + if contest.start_time > timezone_now: + info['subtext'] = {'text': 'upcoming', 'title': str(contest.start_time)} + + for division in contest.info.get('divisions_order', []): + if division not in divisions_order: + divisions_order.append(division) + + if self.score_params.get('regex_problem_name'): + match = re.search(self.score_params.get('regex_problem_name'), contest.title) + if match: + info['short'] = match.group(1) + if self.score_params.get('abbreviation_problem_name'): + info['short'] = ''.join(re.findall(r'(\b[A-Z]|[0-9])', info.get('short', contest.title))) + + problems = contest.info.get('problems', []) + if not detail_problems: + full_score = None + if placing: + if 'division' in placing: + full_score = max([max(p.values()) for p in placing['division'].values()]) + else: + full_score = max(placing.values()) + elif 'division' in problems: + full_scores = [] + for ps in problems['division'].values(): + full = 0 + for problem in ps: + full += problem.get('full_score', 1) + full_scores.append(full) + info['full_score'] = max(full_scores) + elif self.score_params.get('default_problem_full_score'): + full_score = self.score_params['default_problem_full_score'] + else: + full_score = 0 + for problem in problems: + full_score += problem.get('full_score', 1) + if full_score is not None: + info['full_score'] = full_score + problems_infos[str(contest.pk)] = info + else: + for problem in problems: + problem = dict(problem) + add_prefix_to_problem_short(problem, f'{idx}.') + problem['group'] = info.get('short', info['name']) + problem['url'] = info['url'] + problems_infos[get_problem_short(problem)] = problem + + exclude_advances = {} + if advances and advances.get('exclude_stages'): + qs = Statistics.objects \ + .filter(contest__stage__in=advances['exclude_stages'], addition___advance__isnull=False) \ + .values('account__key', 'addition___advance', 'contest__title') \ + .order_by('contest__end_time', 'contest__id') + for r in qs: + d = r['addition___advance'] + if 'contest' not in d: + d['contest'] = r['contest__title'] + exclude_advances[r['account__key']] = d + + statistics = Statistics.objects \ + .select_related('account', 'account__duplicate') \ + .prefetch_related('account__coders') + filter_statistics = self.score_params.get('filter_statistics') + if filter_statistics: + statistics = statistics.filter(**filter_statistics) + exclude_statistics = self.score_params.get('exclude_statistics') + if exclude_statistics: + statistics = statistics.exclude(**exclude_statistics) + re_ranking = self.score_params.get('re_ranking') + + def get_placing(placing, stat): + return placing['division'][stat.addition['division']] if 'division' in placing else placing + + account_keys = dict() + total = statistics.filter(contest__in=contests).count() + with tqdm.tqdm(total=total, desc=f'getting statistics for stage {stage}') as pbar, print_sql(count_only=True): + for idx, contest in enumerate(contests, start=1): + skip_problem_stat = '_skip_for_problem_stat' in contest.info.get('fields', []) + contest_unrated = contest.info.get('unrated') + + if not detail_problems: + problem_info_key = str(contest.pk) + problem_short = get_problem_short(problems_infos[problem_info_key]) + pbar.set_postfix(contest=contest) + stats = statistics.filter(contest_id=contest.pk) + + if placing: + placing_scores = deepcopy(placing) + n_rows = 0 + for s in stats: + n_rows += 1 + placing_ = get_placing(placing_scores, s) + key = str(s.place_as_int) + if key in placing_: + placing_.setdefault('scores', {}) + placing_['scores'][key] = placing_.pop(key) + scores = [] + for place in reversed(range(1, n_rows + 1)): + placing_ = get_placing(placing_scores, s) + key = str(place) + if key in placing_: + scores.append(placing_.pop(key)) + else: + if scores: + placing_['scores'][key] += sum(scores) + placing_['scores'][key] /= len(scores) + 1 + scores = [] + + max_solving = 0 + n_effective = 0 + for stat in stats: + max_solving = max(max_solving, stat.solving) + n_effective += stat.solving > eps + + if re_ranking: + order = contest.get_statistics_order() + stats = stats.order_by(*order) + stat_rank = 0 + stat_last = None + stat_attrs = [attr.strip('-') for attr in order] + + for stat_idx, stat in enumerate(stats, start=1): + if not detail_problems and not skip_problem_stat: + problems_infos[problem_info_key].setdefault('n_total', 0) + problems_infos[problem_info_key]['n_total'] += 1 + + if re_ranking: + stat_value = tuple(get_item(stat, attr) for attr in stat_attrs) + if stat_value != stat_last: + stat_rank = stat_idx + stat_last = stat_value + rank = stat_rank + else: + rank = stat.place_as_int + + score = None + if stat.solving < eps: + score = 0 + if placing: + placing_ = get_placing(placing_scores, stat) + score = placing_.get('zero', 0) + else: + if placing: + placing_ = get_placing(placing_scores, stat) + score = placing_['scores'].get(str(rank), placing_.get('default')) + if score is None: + continue + if scoring: + if score is None: + score = 0 + if scoring['name'] == 'general': + if rank is None: + continue + solving_factor = stat.solving / max_solving + rank_factor = (n_effective - rank + 1) / n_effective + score += scoring['factor'] * solving_factor * rank_factor + elif scoring['name'] == 'field': + score += stat.addition.get(scoring['field'], 0) + else: + raise NotImplementedError(f'scoring {scoring["name"]} is not implemented') + if score is None: + score = stat.solving + + if not detail_problems and not skip_problem_stat: + problems_infos[problem_info_key].setdefault('n_teams', 0) + problems_infos[problem_info_key]['n_teams'] += 1 + if score: + problems_infos[problem_info_key].setdefault('n_accepted', 0) + problems_infos[problem_info_key]['n_accepted'] += 1 + + account = stat.account + if account.duplicate is not None: + account = account.duplicate + + coders = account.coders.all() + has_mapping_account_by_coder = False + if len(coders) == 1: + coder = coders[0] + if coder not in mapping_account_by_coder: + mapping_account_by_coder[coder] = account + else: + account = mapping_account_by_coder[coder] + has_mapping_account_by_coder = True + + row = results[account] + row['member'] = account + account_keys[account.key] = account + + problems = row.setdefault('problems', {}) + if detail_problems: + for key, problem in stat.addition.get('problems', {}).items(): + p = problems.setdefault(f'{idx}.' + key, {}) + if contest_unrated: + p = p.setdefault('upsolving', {}) + p.update(problem) + else: + problem = problems.setdefault(problem_short, {}) + if contest_unrated: + problem = problem.setdefault('upsolving', {}) + problem['result'] = score + url = stat.addition.get('url') + if not url: + url = reverse('ranking:standings_by_id', kwargs={'contest_id': str(contest.pk)}) + url += f'?find_me={stat.pk}' + problem['url'] = url + if contest_unrated: + score = 0 + + if n_best and not detail_problems: + row.setdefault('scores', []).append((score, problem)) + else: + row['score'] = row.get('score', 0) + score + + field_values = {} + for field in fields: + if field.get('skip_on_mapping_account_by_coder') and has_mapping_account_by_coder: + continue + if 'type' in field: + continue + inp = field['field'] + out = field.get('out', inp) + if field.get('first') and out in row or (inp not in stat.addition and not hasattr(stat, inp)): + continue + val = stat.addition[inp] if inp in stat.addition else getattr(stat, inp) + if not field.get('safe') and isinstance(val, str): + val = ast.literal_eval(val) + if 'cast' in field: + val = locate(field['cast'])(val) + field_values[out] = val + if field.get('skip'): + continue + if field.get('accumulate'): + val = round(val + ast.literal_eval(str(row.get(out, 0))), 2) + if field.get('aggregate') == 'avg': + out_n = f'_{out}_n' + out_s = f'_{out}_s' + row[out_n] = row.get(out_n, 0) + 1 + row[out_s] = row.get(out_s, 0) + val + val = round(row[out_s] / row[out_n], 2) + if field.get('aggregate') == 'max' and out in row: + val = max(val, row[out]) + row[out] = val + + if 'solved' in stat.addition and isinstance(stat.addition['solved'], dict): + solved = row.setdefault('solved', {}) + for k, v in stat.addition['solved'].items(): + solved[k] = solved.get(k, 0) + v + + if 'status' in self.score_params: + field = self.score_params['status'] + val = field_values.get(field, row.get(field)) + if val is None: + val = getattr(stat, field) + if val is not None: + problem['status'] = val + else: + for field in order_by: + field = field.lstrip('-') + if field in ['score', 'rating', 'penalty']: + continue + status = field_values.get(field, row.get(field)) + if status is None: + continue + problem['status'] = status + break + pbar.update() + + for writer in contest.writers.all(): + account_keys[writer.key] = writer + + total = sum([len(contest.info.get('writers', [])) for contest in contests]) + with tqdm.tqdm(total=total, desc=f'getting writers for stage {stage}') as pbar, print_sql(count_only=True): + writers = set() + for contest in contests: + contest_writers = contest.info.get('writers', []) + if not contest_writers or detail_problems: + continue + problem_info_key = str(contest.pk) + problem_short = get_problem_short(problems_infos[problem_info_key]) + for writer in contest_writers: + if writer in account_keys: + account = account_keys[writer] + else: + try: + account = Account.objects.get(resource_id=contest.resource_id, key__iexact=writer) + except Account.DoesNotExist: + account = None + + pbar.update() + if not account: + continue + writers.add(account) + + row = results[account] + row['member'] = account + row.setdefault('score', 0) + if n_best: + row.setdefault('scores', []) + row.setdefault('writer', 0) + + row['writer'] += 1 + + problems = row.setdefault('problems', {}) + problem = problems.setdefault(problem_short, {}) + problem['status'] = 'W' + + if self.score_params.get('writers_proportionally_score'): + n_contests = sum(contest.start_time < timezone_now for contest in contests) + for account in writers: + row = results[account] + if n_contests == row['writer'] or 'score' not in row: + continue + row['score'] = row['score'] / (n_contests - row['writer']) * n_contests + if self.score_params.get('exponential_score_decay'): + for r in results.values(): + scores = [problem.get('result', 0) for problem in r.get('problems', {}).values()] + scores.sort(reverse=True) + k = self.score_params['exponential_score_decay'] + score = k * sum((1 - k) ** i * score for i, score in enumerate(scores)) + r['score'] = score + + for field in fields: + t = field.get('type') + if t is None: + continue + if t == 'points_for_common_problems': + group = field['group'] + inp = field['field'] + out = field.get('out', inp) + + excluding = bool(exclude_advances and field.get('exclude_advances')) + + groups = collections.defaultdict(list) + for row in results.values(): + key = row[group] + groups[key].append(row) + + advancement_position = 1 + for key, rows in sorted(groups.items(), key=lambda kv: kv[0], reverse=True): + common_problems = None + for row in rows: + handle = row['member'].key + if excluding and handle in exclude_advances: + exclude_advance = exclude_advances[handle] + for advance in advances.get('options', []): + if advance['next'] == exclude_advance['next']: + exclude_advance['skip'] = True + break + if advancement_position in advance['places']: + break + if exclude_advance.get('skip'): + continue + + problems = {k for k, p in row['problems'].items() if p.get('status') != 'W'} + common_problems = problems if common_problems is None else (problems & common_problems) + if common_problems is None: + for row in rows: + problems = {k for k, p in row['problems'].items() if p.get('status') != 'W'} + common_problems = problems if common_problems is None else (problems & common_problems) + + for row in rows: + handle = row['member'].key + if excluding and not exclude_advances.get(handle, {}).get('skip', False): + advancement_position += 1 + value = 0 + for k in common_problems: + value += float(row['problems'].get(k, {}).get(inp, 0)) + for k, v in row['problems'].items(): + if k not in common_problems and v.get('status') != 'W': + v['status_tag'] = 'strike' + row[out] = round(value, 2) + elif t == 'region_by_country': + out = field['out'] + + mapping_regions = dict() + for regional_event in field['data']: + for region in regional_event['regions']: + mapping_regions[region['code']] = {'regional_event': regional_event, 'region': region} + + for row in results.values(): + country = row['member'].country + if not country: + continue + code = country.code + if code not in mapping_regions: + continue + row[out] = mapping_regions[code]['regional_event']['name'] + elif t == 'n_medal_problems': + for row in results.values(): + for problem in row['problems'].values(): + medal = problem.get('medal') + if medal: + k = f'n_{medal}_problems' + row.setdefault(k, 0) + row[k] += 1 + if k not in hidden_fields: + hidden_fields.append(k) + for field in settings.STANDINGS_FIELDS_: + if field in hidden_fields: + fixed_fields.append(field) + hidden_fields.remove(field) + else: + raise ValueError(f'Unknown field type = {t}') + + hidden_fields += [field.get('out', field.get('inp')) for field in fields if field.get('hidden')] + + results = list(results.values()) + if n_best: + for row in results: + scores = row.pop('scores') + for index, (score, problem) in enumerate(sorted(scores, key=lambda s: s[0], reverse=True)): + if index < n_best: + row['score'] = row.get('score', 0) + score + else: + problem['status'] = problem.pop('result') + + filtered_results = [] + filter_zero_points = self.score_params.get('filter_zero_points', True) + for r in results: + if r['score'] > eps or not filter_zero_points or r.get('writer'): + filtered_results.append(r) + continue + if detail_problems: + continue + + problems = r.setdefault('problems', {}) + + for idx, contest in enumerate(contests, start=1): + skip_problem_stat = '_skip_for_problem_stat' in contest.info.get('fields', []) + if skip_problem_stat: + continue + + problem_info_key = str(contest.pk) + problem_short = get_problem_short(problems_infos[problem_info_key]) + + if problem_short in problems: + problems_infos[problem_info_key].setdefault('n_teams', 0) + problems_infos[problem_info_key]['n_teams'] -= 1 + results = filtered_results + + results = sorted( + results, + key=lambda r: tuple(r.get(k.lstrip('-'), 0) * (-1 if k.startswith('-') else 1) for k in order_by), + reverse=True, + ) + + additions = deepcopy(stage.info.get('additions', {})) + field_to_problem = self.score_params.get('field_to_problem') + with transaction.atomic(): + fields_set = set() + fields = list() + + pks = set() + placing_infos = {} + score_advance = None + place_advance = 0 + place_index = 0 + for row in tqdm.tqdm(results, desc=f'update statistics for stage {stage}'): + for field in [row.get('member'), row.get('name')]: + row.update(additions.pop(field, {})) + + division = row.get('division', 'none') + placing_info = placing_infos.setdefault(division, {}) + placing_info['index'] = placing_info.get('index', 0) + 1 + + curr_score = tuple(row.get(k.lstrip('-'), 0) for k in order_by) + if curr_score != placing_info.get('last_score'): + placing_info['last_score'] = curr_score + placing_info['place'] = placing_info['index'] + + if advances and ('divisions' not in advances or division in advances['divisions']): + tmp = score_advance, place_advance, place_index + + place_index += 1 + if curr_score != score_advance: + score_advance = curr_score + place_advance = place_index + + for advance in advances.get('options', []): + handle = row['member'].key + if handle in exclude_advances and advance['next'] == exclude_advances[handle]['next']: + advance = exclude_advances[handle] + if 'class' in advance and not advance['class'].startswith('text-'): + advance['class'] = f'text-{advance["class"]}' + row['_advance'] = advance + break + + if 'places' in advance and place_advance in advance['places']: + if not advances.get('inplace_fields_only'): + row['_advance'] = advance + if is_over: + for field in advance.get('inplace_fields', []): + row[field] = advance[field] + tmp = None + break + + if tmp is not None: + score_advance, place_advance, place_index = tmp + account = row.pop('member') + solving = row.pop('score') + + advanced = False + if row.get('_advance'): + adv = row['_advance'] + advanced = not adv.get('skip') and not adv.get('class', '').startswith('text-') + + defaults = { + 'place': str(placing_info['place']), + 'place_as_int': placing_info['place'], + 'solving': solving, + 'addition': row, + 'skip_in_stats': True, + 'advanced': advanced, + } + if parsed_statistic: + defaults['place'] = None + defaults['place_as_int'] = None + defaults['solving'] = 0 + stat = Statistics.objects.filter(account=account, contest=stage).first() + if not stat: + continue + for k, v in defaults['addition'].items(): + if k not in stat.addition or (v and not stat.addition.get(k)): + stat.addition[k] = v + stat.skip_in_stats = defaults['skip_in_stats'] + stat.advanced = defaults['advanced'] + stat.save(update_fields=['addition', 'skip_in_stats', 'advanced']) + else: + stat, created = Statistics.objects.update_or_create( + account=account, + contest=stage, + defaults=defaults, + ) + pks.add(stat.pk) + + for k in stat.addition.keys(): + if field_to_problem and re.search(field_to_problem['regex'], k): + continue + if k not in fields_set: + fields_set.add(k) + fields.append(k) + stage.info['problems'] = list(problems_infos.values()) + + if field_to_problem: + for stat in stage.statistics_set.all(): + problems = stat.addition.setdefault('problems', {}) + was_updated = False + for k, v in list(stat.addition.items()): + match = re.search(field_to_problem['regex'], k) + if match: + problem_short = field_to_problem['format'].format(**match.groupdict()) + if problem_short not in problems: + problems[problem_short] = {'result': v} + stat.addition.pop(k) + if 'fields_types' in stage.info: + stage.info['fields_types'].pop(k, None) + was_updated = True + if was_updated: + stat.save(update_fields=['addition']) + + if parsed_statistic: + for field in stage.info.setdefault('hidden_fields', []): + if field not in hidden_fields: + hidden_fields.append(field) + regex_hidden_fields = self.score_params.get('regex_hidden_fields') + if regex_hidden_fields: + for field in fields: + if field not in hidden_fields and re.search(regex_hidden_fields, field): + hidden_fields.append(field) + stage.info['fields'] = list(fields) + stage.info['hidden_fields'] = hidden_fields + + fields_types = self.score_params.get('fields_types', {}) + if fields_types: + stage.info.setdefault('fields_types', {}).update(fields_types) + + if not parsed_statistic: + stage.statistics_set.exclude(pk__in=pks).delete() + stage.n_statistics = len(results) + stage.parsed_time = timezone_now + + standings_info = self.score_params.get('info', {}) + standings_info['fixed_fields'] = fixed_fields + [(f.lstrip('-'), f.lstrip('-')) for f in order_by] + stage.info['standings'] = standings_info + + if divisions_order and self.score_params.get('divisions_ordering'): + stage.info['divisions_order'] = divisions_order + + if stage.is_rated is None: + stage.is_rated = False + stage.save() diff --git a/src/ranking/views.py b/src/ranking/views.py index 98b86994..acd15030 100644 --- a/src/ranking/views.py +++ b/src/ranking/views.py @@ -4,7 +4,6 @@ import re from collections import OrderedDict, defaultdict from functools import reduce -from itertools import accumulate import arrow from asgiref.sync import async_to_sync @@ -50,7 +49,8 @@ @page_template('standings_list_paging.html') @extra_context_without_pagination('clist.view_full_table') -def standings_list(request, template='standings_list.html', extra_context=None): +@context_pagination() +def standings_list(request, template='standings_list.html'): contests = ( Contest.objects.annotate_favorite(request.user) .select_related('resource', 'stage') @@ -89,10 +89,12 @@ def standings_list(request, template='standings_list.html', extra_context=None): 'writer': {'fields': ['info__writers__contains']}, 'coder': {'fields': ['statistics__account__coders__username']}, 'account': {'fields': ['statistics__account__key', 'statistics__account__name'], 'suff': '__iregex'}, - 'stage': {'fields': ['stage'], 'suff': '__isnull', 'func': lambda v: False}, - 'kind': {'fields': ['kind'], 'suff': '__isnull', 'func': lambda v: False}, - 'medal': {'fields': ['with_medals'], 'func': lambda v: True}, - 'advance': {'fields': ['with_advance'], 'func': lambda v: True}, + 'stage': {'fields': ['stage'], 'suff': '__isnull', 'func': lambda v: v not in settings.YES_}, + 'kind': {'fields': ['kind'], 'suff': '__isnull', 'func': lambda v: v not in settings.YES_}, + 'medal': {'fields': ['with_medals'], 'func': lambda v: v in settings.YES_}, + 'related_set': {'fields': ['related_set'], 'suff': '__isnull', + 'func': lambda v: v not in settings.YES_}, + 'advance': {'fields': ['with_advance'], 'func': lambda v: v in settings.YES_}, 'year': {'fields': ['start_time__year', 'end_time__year']}, 'invisible': {'fields': ['invisible'], 'func': lambda v: v in settings.YES_}, 'has_problems': {'fields': ['n_problems'], 'suff': '__isnull', @@ -127,7 +129,10 @@ def standings_list(request, template='standings_list.html', extra_context=None): series = [s for s in request.GET.getlist('series') if s] if series: - series = list(ContestSeries.objects.filter(slug__in=series)) + if 'all' in series: + series = list(ContestSeries.objects.all()) + else: + series = list(ContestSeries.objects.filter(slug__in=series)) link_series = request.GET.get('link_series') in settings.YES_ link_series = link_series and request.user.has_perm('clist.change_contestseries') link_series = link_series and len(series) == 1 @@ -240,10 +245,7 @@ def standings_list(request, template='standings_list.html', extra_context=None): 'contests': contests, }) - if extra_context is not None: - context.update(extra_context) - - return render(request, template, context) + return template, context def _standings_highlight(contest, statistics, options): @@ -546,6 +548,7 @@ def timeline_format(t): problems_chart = dict( field='solved_problems', type='line', + accumulate=True, fields=[], labels={}, bins=problems_bins, @@ -562,7 +565,7 @@ def timeline_format(t): values = problems_values.get(short, []) total_values.extend(values) hist, _ = make_histogram(values=values, bins=problems_bins) - for val, d in zip(accumulate(hist), problems_chart['data']): + for val, d in zip(hist, problems_chart['data']): d[short] = val problems_chart['fields'].append(short) problems_chart['labels'][short] = get_problem_title(problem) @@ -600,7 +603,7 @@ def timeline_format(t): deltas = [v[1] for v in values] values = [v[0] for v in values] hist, _ = make_histogram(values=values, deltas=deltas, bins=problems_bins) - for val, d in zip(accumulate(hist), problems_scoring_chart['data']): + for val, d in zip(hist, problems_scoring_chart['data']): d[short] = val charts.append(problems_scoring_chart) @@ -611,9 +614,10 @@ def timeline_format(t): fields=False, labels=False, my_dataset=None, + accumulate=False, )) hist, _ = make_histogram(values=total_values, bins=problems_bins) - for val, d in zip(accumulate(hist), total_solved_chart['data']): + for val, d in zip(hist, total_solved_chart['data']): d['value'] = val if total_values: charts.append(total_solved_chart) @@ -629,7 +633,7 @@ def timeline_format(t): values = [v[0] for v in total_scoring_values] deltas = [v[1] for v in total_scoring_values] hist, _ = make_histogram(values=values, deltas=deltas, bins=problems_bins) - for val, d in zip(accumulate(hist), total_scoring_chart['data']): + for val, d in zip(hist, total_scoring_chart['data']): d['value'] = val charts.append(total_scoring_chart) @@ -701,7 +705,7 @@ def render_standings_paging(contest, statistics, with_detail=True): statistics = statistics[:per_page] statistics = Statistics.objects.filter(pk__in=statistics) - order = get_statistics_order(contest) + ['pk'] + order = contest.get_statistics_order() + ['pk'] statistics = statistics.order_by(*order) divisions_order = get_standings_divisions_order(contest) @@ -898,21 +902,6 @@ def get_standings_fields(contest, division, with_detail, hidden_fields=None, hid return fields -def get_statistics_order(contest): - options = contest.info.get('standings', {}) - contest_fields = contest.info.get('fields', []).copy() - resource_standings = contest.resource.info.get('standings', {}) - order = copy.copy(options.get('order', resource_standings.get('order'))) - if order: - for f in order: - if f.startswith('addition__') and f.split('__', 1)[1] not in contest_fields: - order = None - break - if order is None: - order = ['place_as_int', '-solving'] - return order - - def get_advancing_contests(contest): ret = set() @@ -1005,7 +994,7 @@ def standings(request, contest, other_contests=None, template='standings.html', per_page = 50 if contests_ids else contest.standings_per_page per_page_more = per_page if find_me else 200 - order = get_statistics_order(contest) + order = contest.get_statistics_order() statistics = statistics \ .select_related('account') \ diff --git a/src/scripts/repack-database.bash b/src/scripts/repack-database.bash new file mode 100755 index 00000000..372b7337 --- /dev/null +++ b/src/scripts/repack-database.bash @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +date + +name=repack_database +exec 200>/tmp/$name.lock +flock -n 200 || { echo "Script '$name' is already running"; exit 0; } + + +function human_readable_size { + local size=$1 + if [ $size -lt 0 ]; then + size=$(($size * -1)) + echo -n "-" + fi + numfmt --to=iec --format="%.2f" $size +} + +table_filter=${1:-"."} +shift + +export PGHOST=$POSTGRES_HOST +export PGPORT=$POSTGRES_PORT +export PGDATABASE=$POSTGRES_DB +export PGUSER=$POSTGRES_USER +export PGPASSWORD=$POSTGRES_PASSWORD + +set -u -e + +psql_command="psql -t -c" +tables=$($psql_command "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname='public';" | grep $table_filter | xargs) +echo "Tables ($(echo $tables | wc -w)): $tables" + +total_saved=0 +for table in $tables; do + table_size=$($psql_command "SELECT pg_total_relation_size('public.$table');" | xargs) + disk_free_size=$(df -B1 / | tail -1 | awk '{print $4}') + threshold_size=$(echo "($disk_free_size * 0.8)/1" | bc | xargs) + echo "Table $table size: $(human_readable_size $table_size)" + echo "Disk free: $(human_readable_size $disk_free_size), threshold size (80%): $(human_readable_size $threshold_size)" + if [ $table_size -gt $threshold_size ]; then + echo "Skipping table $table because it is too big, table size = $(human_readable_size $table_size)" + indices=$($psql_command "SELECT indexname FROM pg_indexes WHERE tablename='$table';") + for index in $indices; do + index_size=$($psql_command "SELECT pg_total_relation_size('public.$index');" | xargs) + if [ $index_size -gt $threshold_size ]; then + echo "Skipping index $index because it is too big, index size = $(human_readable_size $index_size)" + else + pg_repack --index $index + fi + done + else + echo "Repacking (cluster) table $table" + pg_repack --table $table + echo "Repacking (vacuum full) table $table" + pg_repack --table $table --no-order + fi + new_table_size=$($psql_command "SELECT pg_total_relation_size('public.$table');" | xargs) + echo "Table $table new size: $(human_readable_size $new_table_size), saved $(human_readable_size $(($table_size - $new_table_size)))" + total_saved=$(($total_saved + $table_size - $new_table_size)) +done +echo "Total saved: $(human_readable_size $total_saved)" diff --git a/src/scripts/up.bash b/src/scripts/up.bash index 52175dcd..fc06cafa 100755 --- a/src/scripts/up.bash +++ b/src/scripts/up.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e -x +set -e cd "$(dirname "$0")" @@ -13,10 +13,21 @@ function allow_to_proxy() { bridge="br-clist" proxy_port=3128 comment="from $name to proxy" - sudo ufw status numbered | grep "# $comment" | grep -o -E "[0-9]+" | head -n 1 | xargs --no-run-if-empty -I {} sh -c 'yes | sudo ufw delete {}' - sudo ufw allow in on $bridge from $ip to any port $proxy_port comment "$comment" + + existing_rule=$(sudo ufw status numbered | grep "$comment" | grep "$ip") + if [ -n "$existing_rule" ]; then + echo "Rule already exists: $existing_rule" + return + fi + + rule_number=$(sudo ufw status numbered | grep "# $comment" | grep -o -E "[0-9]+" | head -n 1) + if [ -n "$rule_number" ]; then + yes | sudo ufw delete $rule_number + fi + (set -x; sudo ufw allow in on $bridge from $ip to any port $proxy_port comment "$comment") } +allow_to_proxy "prod" allow_to_proxy "dev" allow_to_proxy "legacy" diff --git a/src/static/css/base.css b/src/static/css/base.css index e729bdd1..f9ed5a90 100644 --- a/src/static/css/base.css +++ b/src/static/css/base.css @@ -112,6 +112,21 @@ body { flex: 1; } +.center-container { + display: flex; + justify-content: center; + align-items: center; + height: 100vh; + text-align: center; +} + +.scroll-container { + max-height: 100vh; + overflow-y: auto; + box-sizing: border-box; + padding: 50px; +} + img.resource-icon { vertical-align: top; margin-top: 3px; @@ -825,6 +840,16 @@ a[disabled] { padding: 0px; } +#filter-collapse .input-group { + display: inline-table; + vertical-align: middle; +} + +#filter-collapse .input-group-addon, +#filter-collapse .input-group-btn { + width: auto; +} + #filter-toggle { margin-bottom: 5px; font-size: 10px; @@ -957,3 +982,12 @@ a[data-toggle="tooltip"][disabled], font-size: 80%; cursor: pointer; } + + +/* + * select2 + */ + +.select2-search__field:not([placeholder=""]) { + width: 100% !important; +} diff --git a/src/static/css/form.css b/src/static/css/form.css new file mode 100644 index 00000000..36ff1afc --- /dev/null +++ b/src/static/css/form.css @@ -0,0 +1,25 @@ +#form .btn { + margin-bottom: 15px; +} + +.iframe-panel { + border-radius: 20px; + overflow: hidden; + padding: 40px; +} + +.center-container .scroll-container { + padding: 50px 200px; +} + +@media (max-width: 1200px) { + .center-container .scroll-container { + padding: 50px 100px; + } +} + +@media (max-width: 768px) { + .center-container .scroll-container { + padding: 50px 50px; + } +} diff --git a/src/static/css/settings.css b/src/static/css/settings.css index 20c6ad8c..fc792002 100644 --- a/src/static/css/settings.css +++ b/src/static/css/settings.css @@ -121,6 +121,11 @@ form.editableform .has-error .editable-input .form-control:focus { text-align: right; } +#subscriptions .subscription-account, +#subscriptions .subscription-coder { + margin-right: 10px; +} + .account-suggest { margin-right: 5px; } @@ -129,3 +134,16 @@ form.editableform .has-error .editable-input .form-control:focus { display: inline-block; max-width: 100%; } + +#list-accounts .panel-container { + display: inline-block; + max-width: 100%; +} + +#subscription_form .form-group .toggle { + display: block; +} + +#subscription_form .form-group:has(input:disabled) { + color: #888; +} diff --git a/src/static/css/standings.css b/src/static/css/standings.css index dbdd863d..2a698f55 100644 --- a/src/static/css/standings.css +++ b/src/static/css/standings.css @@ -223,6 +223,10 @@ a.solution:hover span { z-index: 10; } +#table-inner-scroll table tr:not(.info) .sticky-column { + background-color: inherit; +} + .table tr:nth-of-type(n).info td { background-color: #d9edf7; } diff --git a/src/static/img/resources/azspcs_com.png b/src/static/img/resources/azspcs_com.png new file mode 100644 index 00000000..6dec68c1 Binary files /dev/null and b/src/static/img/resources/azspcs_com.png differ diff --git a/src/static/img/resources/calico_cs_berkeley_edu.png b/src/static/img/resources/calico_cs_berkeley_edu.png new file mode 100644 index 00000000..f155e853 Binary files /dev/null and b/src/static/img/resources/calico_cs_berkeley_edu.png differ diff --git a/src/static/img/resources/coliseum_ai.png b/src/static/img/resources/coliseum_ai.png new file mode 100644 index 00000000..d1c41334 Binary files /dev/null and b/src/static/img/resources/coliseum_ai.png differ diff --git a/src/static/img/resources/kattis_com.png b/src/static/img/resources/kattis_com.png new file mode 100644 index 00000000..1f7e3fcc Binary files /dev/null and b/src/static/img/resources/kattis_com.png differ diff --git a/src/static/js/base.js b/src/static/js/base.js index c0598620..c36f57b5 100644 --- a/src/static/js/base.js +++ b/src/static/js/base.js @@ -200,7 +200,7 @@ function log_ajax_error(response, element = null) { message = response.responseJSON.message } } else { - if (response.responseText.length < 100) { + if (response.responseText && response.responseText.length < 100) { message = "{status} {statusText}: {responseText}".format(response) } else { message = "{status} {statusText}, more in console".format(response) @@ -213,6 +213,7 @@ function log_ajax_error(response, element = null) { } else { $.notify(message, 'error') } + $('.bootbox').effect('shake') } function log_ajax_error_callback(response) { @@ -657,7 +658,7 @@ function coders_select(id, submit) { data: function (params) { return { query: 'coders', - regex: params.term, + search: params.term, page: params.page || 1 } }, @@ -781,3 +782,24 @@ function is_12_hour_clock() { var time_parts = date_time_format.formatToParts(new Date()) return time_parts.some(time_part => time_part.type === 'dayPeriod') } + + +/* + * select2 + */ + +$(() => { + $('[data-init="select2"]').select2({theme: 'bootstrap', dropdownAutoWidth: true}) +}) + + +function show_extra(element) { + var $element = $(element) + var extra_id = $element.data('id') + var $extra = $('#' + extra_id) + $extra.toggleClass('hidden') + + var text = $element.data('toggle-text') || 'hide' + $element.data('toggle-text', $element.text()) + $element.text(text) +} diff --git a/src/static/js/contest/calendar.js b/src/static/js/contest/calendar.js index c49fe31d..149753fb 100644 --- a/src/static/js/contest/calendar.js +++ b/src/static/js/contest/calendar.js @@ -117,6 +117,10 @@ $(function() { .toArray(), }, success: function (response) { + response.forEach(function(event) { + event.original_start = event.start + event.original_end = event.end + }) successCallback(response) }, error: function(response) { @@ -128,8 +132,8 @@ $(function() { eventDidMount: function (info) { var event = info.event var element = info.el - var start = FullCalendar.Moment.toMoment(event.start, calendar) - var end = FullCalendar.Moment.toMoment(event.end, calendar) + var start = FullCalendar.Moment.toMoment(event.extendedProps.original_start, calendar) + var end = FullCalendar.Moment.toMoment(event.extendedProps.original_end, calendar) var now = FullCalendar.Moment.toMoment($.now(), calendar) var countdown = event.extendedProps.countdown var start_time = start.format('YYYY-MM-DD HH:mm') diff --git a/src/static/js/contest/main.js b/src/static/js/contest/main.js index a702a4a1..9c667f8a 100644 --- a/src/static/js/contest/main.js +++ b/src/static/js/contest/main.js @@ -61,7 +61,7 @@ $(function() { var modal = $(this) modal.find('.modal-title').text(title) modal.find('[name="contest_id"]').val(contest_id) - modal.find('select option[value="' + method + '"]').attr('selected', 'true') + modal.find('select option[value="' + method + '"]').attr('selected', 'true').trigger('change') }) $('#send_notification form').submit(function(e) { @@ -74,7 +74,8 @@ $(function() { }).done(function() { form.reset(); $('#send_notification').modal('hide') - }).fail(function() { + }).fail(function(response) { + log_ajax_error(response) $('#send_notification').effect("shake") }); e.preventDefault(); diff --git a/src/static/js/settings.js b/src/static/js/settings.js index 85bc67e8..57080502 100644 --- a/src/static/js/settings.js +++ b/src/static/js/settings.js @@ -639,7 +639,7 @@ $(function() { var $div = $('\ <div> \ <a href="#" class="filter" data-name="filter" data-value=\'' + JSON.stringify(data) + '\'></a> \ - <a href="#" data-id="' + data.id + '" data-action="delete-filter" data-success="$div.remove();" class="action-filter btn btn-default btn-xs"> \ + <a href="#" data-id="' + data.id + '" data-action="delete-filter" data-success="$element.remove();" class="action-filter btn btn-default btn-xs"> \ <i class="far fa-trash-alt"></i> \ </a> \ </div> \ @@ -894,51 +894,150 @@ $(function() { function process_subscription() { name = $(this).attr('data-name') + data = $(this).data('form') var form = $(` - <form id="process_subscription-form"> + <h3>` + (data? 'Edit' : 'Create') + ` subscription</h3> + <form id="subscription_form"> + <div class="form-group"> + <label class="control-label">Resource</label> + <select class="form-control" name="resource"></select> + </div> <div class="form-group"> <label class="control-label">Contest</label> - <select class="form-control" name="contest" required></select> - <small class="form-text text-muted">Required field</small> + <select class="form-control" name="contest"></select> + </div> + <div class="form-group"> + <label class="control-label">First accepted</label> + <input type="checkbox" data-toggle="toggle" data-on="On" data-off="Off" data-onstyle="default" data-offstyle="default" data-size="normal" name="with_first_accepted"> + </div> + <div class="form-group"> + <label class="control-label">Top n</label> + <input type="number" class="form-control" name="top_n" min="1" max="` + SUBSCRIPTION_TOP_N_LIMIT + `"> + </div> + <div class="form-group"> + <label class="control-label">Accounts <a href="` + ACCOUNTS_URL + `" target="_blank">` + EXTRA_URL_ICON + `</a></label> + <select class="form-control" name="accounts" multiple></select> + <small class="form-text text-muted">Limit ` + SUBSCRIPTION_N_LIMIT + ` accounts</small> + </div> + <div class="form-group"> + <label class="control-label">Coders <a href="` + CODERS_URL + `" target="_blank">` + EXTRA_URL_ICON + `</a></label> + <select class="form-control" name="coders" multiple></select> + <small class="form-text text-muted">Limit ` + SUBSCRIPTION_N_LIMIT + ` coders</small> + </div> + <div class="form-group"> + <label class="control-label">List <a href="` + LISTS_URL + `" target="_blank">` + EXTRA_URL_ICON + `</a></label> + <select class="form-control" name="list"></select> + <small class="form-text text-muted">Use coders and accounts from updated list</small> </div> <div class="form-group"> - <label class="control-label">Account</label> - <select class="form-control" name="account" required></select> - <small class="form-text text-muted">Will be available after selecting a contest. Required field</small> + <label class="control-label">Chat <a href="` + CHATS_URL + `" target="_blank">` + EXTRA_URL_ICON + `</a></label> + <select class="form-control" name="chat"></select> + <small class="form-text text-muted">Use coders and accounts from updated chat</small> </div> <div class="form-group"> <label class="control-label">Method</label> <select class="form-control" name="method" required></select> - <small class="form-text text-muted">Required field</small> + <small class="form-text text-muted">Notification method</small> </div> + <input type="hidden" name="no_stage" value="true"> </form> `); + function set_disabled() { + var disable_addition = !$select_resource.val() && !$select_contest.val() + $with_first_accepted.prop('disabled', disable_addition) + $top_n.prop('disabled', disable_addition) + + var disabled_chat_and_list = $select_accounts.val().length || $select_coders.val().length + var disabled_accounts_and_coders = $select_coder_list.val() || $select_coder_chat.val() + $select_coders.prop('disabled', disabled_accounts_and_coders) + $select_accounts.prop('disabled', disabled_accounts_and_coders) + $select_coder_chat.prop('disabled', $select_coder_list.val() || disabled_chat_and_list) + $select_coder_list.prop('disabled', $select_coder_chat.val() || disabled_chat_and_list) + } + + var $select_resource = form.find('select[name=resource]') + $select_resource.select2({ + dropdownAutoWidth : true, + width: '100%', + theme: 'bootstrap', + placeholder: 'Select resource', + ajax: select2_ajax_conf('resources', 'regex'), + minimumInputLength: 0, + allowClear: true, + }) + $select_resource.on('change', set_disabled) + + var $select_no_stage = form.find('input[name=no_stage]') var $select_contest = form.find('select[name=contest]') $select_contest.select2({ dropdownAutoWidth : true, width: '100%', theme: 'bootstrap', placeholder: 'Select contest', - ajax: select2_ajax_conf('contest-for-add-subscription', 'regex'), + ajax: select2_ajax_conf('contests', 'regex', {resource: $select_resource, 'no_stage': $select_no_stage}), + minimumInputLength: 0, + allowClear: true, + }) + $select_contest.on('change', set_disabled) + + var $with_first_accepted = form.find('input[name=with_first_accepted]') + $with_first_accepted.bootstrapToggle() + $with_first_accepted.prop('disabled', true) + + var $top_n = form.find('input[name=top_n]') + $top_n.prop('disabled', true) + + var $select_accounts = form.find('select[name=accounts]') + $select_accounts.select2({ + dropdownAutoWidth : true, + width: '100%', + theme: 'bootstrap', + placeholder: 'Select accounts', + ajax: select2_ajax_conf('accounts', 'search', {contest: $select_contest, resource: $select_resource}), minimumInputLength: 0, + allowClear: true, + multiple: true, }) - $select_contest.on('change', () => { - $select_account.prop('disabled', false) - $select_account.val(null).trigger('change') + $select_accounts.on('change', set_disabled) + + var $select_coders = form.find('select[name=coders]') + $select_coders.select2({ + dropdownAutoWidth : true, + width: '100%', + theme: 'bootstrap', + placeholder: 'Select coders', + ajax: select2_ajax_conf('coders', 'search', {contest: $select_contest, resource: $select_resource}), + minimumInputLength: 0, + allowClear: true, + multiple: true, + }) + $select_coders.on('change', set_disabled) + + var $select_coder_list = form.find('select[name=list]') + $select_coder_list.select2({ + dropdownAutoWidth : true, + width: '100%', + theme: 'bootstrap', + placeholder: 'Select coder list', + ajax: select2_ajax_conf('coder_lists', 'search'), + minimumInputLength: 0, + allowClear: true, }) + $select_coder_list.on('change', set_disabled) - var $select_account = form.find('select[name=account]') - $select_account.select2({ + var $select_coder_chat = form.find('select[name=chat]') + $select_coder_chat.select2({ dropdownAutoWidth : true, width: '100%', theme: 'bootstrap', - placeholder: 'Select account', - ajax: select2_ajax_conf('account-for-add-subscription', 'search', {contest: $select_contest}), - disabled: true, + placeholder: 'Select coder chat', + ajax: select2_ajax_conf('coder_chats', 'search'), minimumInputLength: 0, + allowClear: true, }) + $select_coder_chat.on('change', set_disabled) var $select_method = form.find('select[name=method]') $select_method.select2({ @@ -948,37 +1047,80 @@ $(function() { placeholder: 'Select method', }) + if (data) { + if (data.resource) { + $select_resource.select2('trigger', 'select', {data: data.resource}) + } + if (data.contest) { + $select_contest.select2('trigger', 'select', {data: data.contest}) + } + if (data.coder_list) { + $select_coder_list.select2('trigger', 'select', {data: data.coder_list}) + } else if (data.coder_chat) { + $select_coder_chat.select2('trigger', 'select', {data: data.coder_chat}) + } else { + if (data.accounts) { + data.accounts.forEach(option => { + $select_accounts.append(new Option(option.text, option.id, true, true)).trigger('change') + }) + $select_accounts.trigger('change') + } + if (data.coders) { + data.coders.forEach(option => { + $select_coders.append(new Option(option.text, option.id, true, true)).trigger('change') + }) + $select_coders.trigger('change') + } + } + if (data.with_first_accepted) { + $with_first_accepted.bootstrapToggle('on') + } + if (data.top_n) { + $top_n.val(data.top_n) + } + if (data.method) { + $select_method.select2('trigger', 'select', {data: data.method}) + } + } + bootbox.confirm(form, function(result) { if (!result) { return } - var form = document.getElementById('process_subscription-form') - if (!form.checkValidity()) { - form.reportValidity() - return false - } + + $('.bootbox-accept').attr('disabled', 'disabled') $.ajax({ type: 'POST', url: $.fn.editable.defaults.url, data: { pk: $.fn.editable.defaults.pk, name: name, + resource: $select_resource.val(), contest: $select_contest.val(), - account: $select_account.val(), + accounts: $select_accounts.val(), + coders: $select_coders.val(), + coder_list: $select_coder_list.val(), + coder_chat: $select_coder_chat.val(), method: $select_method.val(), + with_first_accepted: $with_first_accepted.prop('checked'), + top_n: $top_n.val(), + + ...(data && data.id ? {id: data.id} : {}), }, - success: function(data) { - window.location.replace(SUBSCRIPTIONS_URL) - }, - error: function(response) { + success: (data) => window.location.replace(SUBSCRIPTIONS_URL), + error: (response) => { + $('.bootbox-accept').removeAttr('disabled') log_ajax_error(response) }, }) + + return false }) $('.bootbox.modal').removeAttr('tabindex') } $('#add-subscription').click(process_subscription) + $('.edit-subscription').click(process_subscription) var ntf_form = $('#notification-form') var ntf_add = $('#add-notification') @@ -1036,10 +1178,12 @@ $(function() { function sentAction() { var $this = $(this) - var $div = $this.parent() + var element_closest = $this.attr('data-closest-element') + var $element = element_closest? $this.closest(element_closest) : $this.parent() var url = $this.attr('data-url') || $.fn.editable.defaults.url var name = $this.attr('data-name') || 'name' var type = $this.attr('data-type') || 'POST' + var dialog = undefined var data = { pk: $.fn.editable.defaults.pk, @@ -1053,28 +1197,37 @@ $(function() { } function queryAction() { + if (dialog) dialog.find('.bootbox-accept').attr('disabled', 'disabled') $.ajax({ type: type, url: url, data: data, success: function(data) { eval($this.attr('data-success')) + if (dialog) dialog.modal('hide') }, - error: function(data) { - $.notify("{status} {statusText}: {responseText}".format(data), "error"); - }, + error: (response) => log_ajax_error(response), + complete: () => { + if (dialog) dialog.find('.bootbox-accept').removeAttr('disabled') + } }) } if ($this.attr('data-confirm') === 'false') { queryAction() } else { - bootbox.confirm({ + dialog = bootbox.confirm({ size: 'small', - message: $div.text() + + message: $element.text() + "<br/><br/>" + "<b>" + $this.attr('data-action').replace('-', ' ').toTitleCase() + "?</b>", - callback: function(result) { if (result) { queryAction() } }, + callback: (result) => { + if (!result) { + return + } + queryAction() + return false + }, }) } return false @@ -1097,8 +1250,8 @@ $(function() { $('#first-name-native').editable({ type: 'text', }) $('#last-name-native').editable({ type: 'text', }) - var $resource = $('select#add-account-resource') - $resource.select2({ + var $search_resource = $('select#add-account-resource') + $search_resource.select2({ width: '40%', allowClear: true, placeholder: 'Search resource by regex', @@ -1158,23 +1311,12 @@ $(function() { }) } - function addAccount(index, element) { - var $block = $('<div class="account">') - .append($('<a>', {class: 'delete-account btn btn-default btn-xs'}).attr('data-id', element.pk).append($('<i>', {class: 'far fa-trash-alt'}))) - .append($('<span>', {text: ' '})) - .append($('<span>', {text: element.account + (element.name && element.account.indexOf(element.name) == -1? ' | ' + element.name : '')})) - .append($('<span>', {text: ' '})) - .append($('<a>', {class: 'text-muted small', href: 'http://' + element.resource, text: element.resource})) - - $block.find('.delete-account').click(deleteAccount) - $listAccount.prepend($block) - } $('.delete-account').click(deleteAccount) var $account_suggests = $('#account-suggests') function selectAccountSuggest() { - $search.val($(this).data('handle')).trigger('change') + $search_account.val($(this).data('handle')).trigger('change') $account_suggests.children().remove() } @@ -1188,46 +1330,64 @@ $(function() { $account_suggests.append($suggest) } - var $search = $('#add-account-search') - $search.css({'width': '40%'}); + var $search_account = $('#add-account-search') + $search_account.css({'width': '40%'}); var $add_account_button = $('#add-account') var $add_account_loading = $('#add-account-loading') function update_advanced_search() { + $add_account_button.prop('disabled', !($search_resource.val() && $search_account.val())) + $advanced_search = $('#add-account-advanced-search') href = ACCOUNTS_ADVANCED_SEARCH_URL - if ($resource.val()) { - href += '&resource=' + $resource.val() + if ($search_resource.val()) { + href += '&resource=' + $search_resource.val() } - if ($search.val()) { - href += '&search=' + encodeURIComponent($search.val()) + if ($search_account.val()) { + href += '&search=' + encodeURIComponent($search_account.val()) } $advanced_search.attr('href', href) } - $resource.on('change', update_advanced_search) - $search.on('keyup', update_advanced_search) + + $search_resource.on('change', update_advanced_search) + $search_account.on('keyup', update_advanced_search) update_advanced_search() + function filter_account_table() { + var resource_pk = $search_resource.val() + if (resource_pk) { + $('#list-accounts .account').addClass('hidden') + $('#list-accounts .account[data-account-resource="' + resource_pk + '"]').removeClass('hidden') + } else { + $('#list-accounts .account').removeClass('hidden') + } + } + $search_resource.on('change', filter_account_table) + filter_account_table() + $add_account_button.click(function() { $add_account_loading.removeClass('hidden') + $add_account_button.prop('disabled', true) $.ajax({ type: 'POST', url: $.fn.editable.defaults.url, data: { pk: $.fn.editable.defaults.pk, name: 'add-account', - resource: $resource.val(), - value: $search.val(), + resource: $search_resource.val(), + value: $search_account.val(), }, success: function(data) { - $add_account_loading.addClass('hidden') - $account_suggests.children().remove() if (data.message == 'add') { - addAccount(-1, data.account) - $resource.val(null).trigger('change') - $search.val(null).trigger('change') - } else if (data.message == 'suggest') { + window.location.replace(ACCOUNTS_TAB_URL + '?resource=' + $search_resource.val()) + return + } + + $add_account_loading.addClass('hidden') + $add_account_button.prop('disabled', false) + if (data.message == 'suggest') { + $account_suggests.children().remove() for (var i = 0; i < data.accounts.length; i++) { addAccountSuggest(data.accounts[i]) } @@ -1239,13 +1399,14 @@ $(function() { return } $add_account_loading.addClass('hidden') + $add_account_button.prop('disabled', false) $errorAccountTab.show().html(data.responseText) setTimeout(function() { $errorAccountTab.hide(500) }, 3000) }, }) }) - $search.keypress(function(e) { + $search_account.keypress(function(e) { if (e.which == 13 ) { e.preventDefault() $add_account_button.click() diff --git a/src/static/js/standings.js b/src/static/js/standings.js index b2aa01ac..7d19b452 100644 --- a/src/static/js/standings.js +++ b/src/static/js/standings.js @@ -256,8 +256,9 @@ function process_problem_cell(element, current_time, unfreeze_index, percentage_ } var visible = true + var is_active_switcher = stat.attr('data-active-switcher') - if (stat.attr('data-active-switcher')) { + if (is_active_switcher) { if (problem_submission) { data_penalty = Math.floor(problem_submission[0] / 60) data_score = problem_submission[1] @@ -311,14 +312,18 @@ function process_problem_cell(element, current_time, unfreeze_index, percentage_ var time = 0 console.log('Unknown problem time') } - visible = unfreezing || time <= current_time + to_show = time <= current_time + visible = unfreezing || to_show if (unfreezing || !is_virtual_start) { - if (time > unfreeze_duration() && !unfreeze_open) { + if ( + !unfreeze_open && time > unfreeze_duration() && !is_hidden(score) && !is_active_switcher || + !unfreeze_open && unfreezing && is_active_switcher + ) { score = '?' visible = false } - if (!with_virtual_start && time <= current_time) { + if (!with_virtual_start && to_show) { stat.addClass('result-question') } else { stat.removeClass('result-question') @@ -466,8 +471,7 @@ function prepare_unfreeze() { stat_cells.sort(cmp_row) - var last_problem_cell = null - var candidates_selector = '.problem-cell.problem-cell-stat.result-unfreeze' + var candidates_selector = '.problem-cell.problem-cell-stat.result-unfreeze:not(.prepered-unfreeze)' var has_empty_penalty = false $(candidates_selector).each((_, e) => has_empty_penalty |= !$(e).attr('data-penalty')) @@ -491,17 +495,8 @@ function prepare_unfreeze() { } }) - if (last_problem_cell == problem_cell) { - bootbox.alert({ - message: 'Something went wrong while building the unfreezing.', - className: 'text-danger text-weight-bold', - backdrop: true, - }); - return - } - last_problem_cell = problem_cell - var opening = $(problem_cell) + opening.addClass('prepered-unfreeze') var stat = opening.parent('.stat-cell') var statistic_id = stat.attr('data-statistic-id') var problem_key = opening.attr('data-problem-key') @@ -519,6 +514,7 @@ function prepare_unfreeze() { } } } + $('.prepered-unfreeze').removeClass('prepered-unfreeze') } function change_freeze_duration(select) { diff --git a/src/templates/account_table_cell.html b/src/templates/account_table_cell.html index eabef8d9..8b4521b9 100644 --- a/src/templates/account_table_cell.html +++ b/src/templates/account_table_cell.html @@ -60,7 +60,7 @@ </span> {% if not without_url and not without_inline_url %} -<span class="inline-button"> +<span{% if not without_inline_button %} class="inline-button"{% endif %}> {% if not without_profile_url %} {% profile_url account resource=resource %} {% endif %} diff --git a/src/templates/base.html b/src/templates/base.html index 078fa263..33242b65 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -7,6 +7,7 @@ <meta name="description" property="og:description" content="{% block description %}Join us on a journey into the world of competitive programming. Our platform keeps you updated about past events and future contests. Get personalized alerts for contests you're interested in and track the progress of coders worldwide. Learn from others, improve your coding skills, and become part of an active community of programmers. Begin your programming adventure with us today.{% endblock %}{% include "filter_title.html" with pretext="For" %}"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> + {% if not nofavicon %} <link rel="apple-touch-icon" sizes="180x180" href="{% static_ts 'img/favicon/apple-touch-icon.png' %}"> <link rel="icon" type="image/png" sizes="32x32" href="{% static_ts 'img/favicon/favicon-32x32.png' %}"> <link rel="icon" type="image/png" sizes="16x16" href="{% static_ts 'img/favicon/favicon-16x16.png' %}"> @@ -15,6 +16,7 @@ <meta name="msapplication-TileColor" content="#2b5797"> <meta name="theme-color" content="#f8f8f8"> <meta name="msapplication-config" content="{% static_ts 'img/favicon/browserconfig.xml' %}"> + {% endif %} {% if user.is_authenticated %} <meta name="sw" content="{% static_ts 'js/sw.js' %}"> @@ -24,9 +26,9 @@ <meta property="og:title" content="{% block full_ogtitle %}{% block ogtitle %}{% endblock %}{% include "filter_title.html" with pretext="for" %} - CLIST{% endblock %}"> <title>{% block full_title %}{% block title %}{% endblock %}{% include "filter_title.html" with pretext="for" %} - CLIST{% endblock %} - + - {% if not DEBUG %} + {% if not DEBUG and not nocounter %} {% include 'counter/base.html' %} {% endif %} @@ -92,6 +94,7 @@ {% block end-head %}{% endblock %} + {% block body %} {% endif %} + {% endblock %} diff --git a/src/templates/chart.html b/src/templates/chart.html index 135b9294..d9e1454c 100644 --- a/src/templates/chart.html +++ b/src/templates/chart.html @@ -1,44 +1,51 @@ {% if chart %} +{% define chart.type|default:"bar" as chart_type %}
    {% if not without_buttons %}
    {% if not without_toggle_accumulate %} + {% if chart_type == 'line' or chart_type == 'bar' or chart.accumulate %} + type="checkbox" + data-toggle="toggle" + data-on="accumulate" + data-off="normal" + data-onstyle="default active" + data-offstyle="default active" + data-size="mini" + {% if chart.accumulate %}checked{% endif %} + /> {% endif %} + {% endif %} + type="checkbox" + data-toggle="toggle" + data-on="logarithmic" + data-off="linear" + data-onstyle="default active" + data-offstyle="default active" + data-size="mini" + /> {% if not without_toggle_type %} + {% if chart_type == 'line' or chart_type == 'bar' %} + type="checkbox" + data-toggle="toggle" + data-on="line" + data-off="bar" + data-onstyle="default active" + data-offstyle="default active" + data-size="mini" + {% if chart_type == 'line' %}checked{% endif %} + /> {% endif %} + {% endif %} {% if expand_class %} @@ -170,7 +177,7 @@ } var config = { - type: '{{ chart.type|default:"bar" }}', + type: '{{ chart_type }}', data: data, options: { responsive: true, @@ -422,7 +429,7 @@ original_data = null } chart.update() - }) + }){% if chart.accumulate %}.prop('checked', true).change(){% endif %} $('#chart_{{ chart.field }} .toggle-log-scale').change(function() { chart.options.scales.y.type = chart.options.scales.y.type == 'linear'? 'logarithmic' : 'linear' diff --git a/src/templates/coder.html b/src/templates/coder.html new file mode 100644 index 00000000..b61a700e --- /dev/null +++ b/src/templates/coder.html @@ -0,0 +1,12 @@ +{% if with_fixed_width %}
    {% endif %} +{% get_country_from_coder coder as country %} +{% if country %} + +
    +     +{% endif %} +{% if with_fixed_width %}
    {% endif %} + + + {{ coder.display_name }} +     diff --git a/src/templates/coder_list.html b/src/templates/coder_list.html index affc0a4d..5939b36b 100644 --- a/src/templates/coder_list.html +++ b/src/templates/coder_list.html @@ -78,9 +78,9 @@

    {% for v in data.list_values %} {% if forloop.counter0 %}|{% endif %} {% if v.coder %} - {{ v.coder.username }} + {% include "coder.html" with coder=v.coder with_fixed_width=True %} {% elif v.account %} - {% include "account_table_cell.html" with resource=v.account.resource account=v.account with_resource=True %} + {% include "account_table_cell.html" with resource=v.account.resource account=v.account with_resource=True with_fixed_width=True %} {% else %} — {% endif %} diff --git a/src/templates/event.html b/src/templates/event.html index 17a5151d..03692821 100644 --- a/src/templates/event.html +++ b/src/templates/event.html @@ -1,6 +1,4 @@ {% extends "base.html" %} -{% load jsonify %} - {% block end-head %} diff --git a/src/templates/field_value.html b/src/templates/field_value.html index ee156f01..acc58b5b 100644 --- a/src/templates/field_value.html +++ b/src/templates/field_value.html @@ -80,6 +80,8 @@ {% for handle in value %}{% if forloop.counter0 %}
    {% endif %}{% if resource %}{{ handle }}{% else %}{{ handle }}{% endif %}{% endfor %} {% elif field == 'hints' and types|to_json == '["list"]' %}
      {% for hint in value %}
    • {{ hint }}
      • {% endfor %}
    + {% elif value|get_type == 'RelatedManager' %} + {% for object in value.all %}{{ object }}{% endfor %} {% else %} {{ template_value }} {% endif %} diff --git a/src/templates/form.html b/src/templates/form.html new file mode 100644 index 00000000..0825e086 --- /dev/null +++ b/src/templates/form.html @@ -0,0 +1,40 @@ +{% extends "base.html" %} + +{% block full_ogtitle %}{% block full_title %}{{ form.name|striptags }} - Form - CLIST{% endblock %}{% endblock %} +{% block end-head %}{% endblock %} +{% block favicon %}{% endblock %} +{% block counter %}{% endblock %} + +{% block body %} +
    +
    +

    + {{ form.name|safe }} +

    + {% if form.is_closed %} +
    This form is closed
    + {% else %} + {% if token %} + +
    +
    + {{ code|safe }} +
    +
    + {% else %} + + {% endif %} + + {% if form.end_time %} +
    + Form will close in {{ form.end_time|hr_timedelta }} +
    + {% endif %} + {% endif %} +
    +
    +{% endblock %} diff --git a/src/templates/main.html b/src/templates/main.html index d9a10343..8002601d 100644 --- a/src/templates/main.html +++ b/src/templates/main.html @@ -90,6 +90,9 @@ {{ time_field|title }} time {{ time_field|title }}s in Duration + {% for field in more_fields %} + {% with title_field=field|title_field %}{% for f in title_field.split %}{% if forloop.counter0 %}
    {% endif %}{{ f }}{% endfor %}{% endwith %} + {% endfor %} {% if hide_contest %} @@ -109,7 +112,7 @@ {% with contest_time_field=time_field|add:"_time" %} - + {{ contest|get_item:contest_time_field|timezone:timezone|format_time:time_format }} {% endwith %} @@ -119,7 +122,19 @@ {% if contest.is_over %}over{% else %}{{ next_time|countdown }}{% endif %} {% endwith %} - {{ contest.hr_duration }} + + {{ contest.hr_duration }} + {% if contest.is_coming and contest.with_virtual_start %} + {% icon_to "period" %} + {% endif %} + + {% for field in more_fields %} + + {% with value=contest|get_item:field %} + {% if value is None %}·{% else %}{{ value }}{% endif %} + {% endwith %} + + {% endfor %} {% if hide_contest %} @@ -156,7 +171,7 @@ {{ contest.group_size }}  {% endif %} - {% include "contest_inline_buttons.html" with user=request.user contest=contest only %} + {% include "contest_inline_buttons.html" with user=request.user contest=contest perms=perms only %}
    {{ contest.host }}
    diff --git a/src/templates/main_notification.html b/src/templates/main_notification.html index fcdfdda2..ba26148d 100644 --- a/src/templates/main_notification.html +++ b/src/templates/main_notification.html @@ -1,9 +1,9 @@
    - {% for k, v in request.user.coder.get_notifications %} - + {% endfor %}
    diff --git a/src/templates/message/telegram b/src/templates/message/telegram index c2b4aeb8..4c79b069 100644 --- a/src/templates/message/telegram +++ b/src/templates/message/telegram @@ -1,5 +1,5 @@ {% autoescape off %} {% if prefix %}*{{ prefix }}* -{% endif %}{% for c in contests %}[{{ c.title|md_escape }}]({{ c.url }}) `{{ c.host }}`{% if c.is_over %} is over{% else %} will {% if c.is_running %}end{% else %}start{% endif %} in [{{ c.next_time|hr_timedelta }}](https://www.timeanddate.com/worldclock/fixedtime.html?msg={{ c.title|urlencode }}&iso={% if c.is_running %}{{ c.end_time|format_time:"%Y%m%dT%H%M" }}{% else %}{{ c.start_time|format_time:"%Y%m%dT%H%M" }}{% endif %}{% if c.duration|less_24_hours %}&ah={{ c.duration|hours }}&am={{ c.duration|minutes }}{% endif %}){% endif %} +{% endif %}{% for c in contests %}[{{ c.title|md_url_text }}]({{ c.actual_url }}) `{{ c.host }}`{% if c.is_over %} ended{% else %} will {% if c.is_running %}end{% else %}start{% endif %} in{% endif %} [{% if c.is_over %}{{ c.end_time|naturaltime }}{% else %}{{ c.next_time|hr_timedelta }}{% endif %}](https://www.timeanddate.com/worldclock/fixedtime.html?msg={{ c.title|urlencode }}&iso={% if c.is_running or c.is_over %}{{ c.end_time|format_time:"%Y%m%dT%H%M" }}{% else %}{{ c.start_time|format_time:"%Y%m%dT%H%M" }}{% if c.duration|less_24_hours %}&ah={{ c.duration|hours }}&am={{ c.duration|minutes }}{% endif %}{% endif %}) {% endfor %} {% endautoescape %} diff --git a/src/templates/navbar.html b/src/templates/navbar.html index 00aecf30..6263f0c9 100644 --- a/src/templates/navbar.html +++ b/src/templates/navbar.html @@ -107,6 +107,7 @@
  •  Lists
  •  Notifications
  •  Calendars
    • +
  • {% icon_to 'subscription' '' %} Subscriptions
  • Tools
  •  API
    • diff --git a/src/templates/party.html b/src/templates/party.html index 004dd0d9..028c9688 100644 --- a/src/templates/party.html +++ b/src/templates/party.html @@ -26,7 +26,7 @@

    Coders - {{ party.coders.all.count }} + {{ coders|length }}

    {% for coder in coders %} @@ -46,18 +46,26 @@

    Contests
    Not set contest for party
    {% endfor %}

    - - - {% if party.author == request.user.coder %} - [Join] - [Leave] + {% if party.author == request.user.coder or request.user.coder in admins %} +
    + Copy join url + Copy leave url + {% if admins %} +

    Admins + {{ admins|length }} +

    +
    + {% for coder in admins %} + {{ coder.username }} + {% endfor %} +
    + {% endif %} {% endif %} -
    {% if user and user.is_authenticated %}{% if not user.first_name or not user.last_name %}
    - Fill in your first last and last name in social settings. + Fill in your first last and last name in social settings.
    {% endif %}{% endif %} @@ -75,7 +83,7 @@

    Contests
    - + {{ contest.title|truncatechars:33 }}
    {% if contest.is_coming %}starts{% else %}ends{% endif %} in {{ contest.next_time|countdown }}
    diff --git a/src/templates/profile.html b/src/templates/profile.html index 21377372..8b66d741 100644 --- a/src/templates/profile.html +++ b/src/templates/profile.html @@ -175,7 +175,7 @@

    Ratings{% if two_columns %}{{ history_resources|length|substract:history_resources_limit }} more + {{ history_resources|length|subtract:history_resources_limit }} more
    {% endif %}
    diff --git a/src/templates/profile_account.html b/src/templates/profile_account.html index ea08ae23..69a42424 100644 --- a/src/templates/profile_account.html +++ b/src/templates/profile_account.html @@ -26,7 +26,12 @@

    {% endif %} {% coder_color_circle account.resource account size=28 %} - {% if account.resource.info.standings.name_instead_key and account.name or account.info|get_item:"_name_instead_key" and account.name %}{{ account.name }}{% else %}{{ account.key }}{% endif %} + {% if account.deleted %} + {% define "strike" as account_tag %} + {% else %} + {% define "span" as account_tag %} + {% endif %} + <{{ account_tag }} class="{% coder_color_class account.resource account.info %}">{% if account.resource.info.standings.name_instead_key and account.name or account.info|get_item:"_name_instead_key" and account.name %}{{ account.name }}{% else %}{{ account.key }}{% endif %} {% profile_url account %} diff --git a/src/templates/profile_contests_paging.html b/src/templates/profile_contests_paging.html index b1e4581c..f95d0f57 100644 --- a/src/templates/profile_contests_paging.html +++ b/src/templates/profile_contests_paging.html @@ -11,7 +11,7 @@ {% with percent=statistic.place_as_int|multiply:100|divide:n_total %}
    Rank: {{ statistic.place }}
    Total: {{ n_total }}" data-toggle="tooltip" data-placement="top" data-html="true"> -
    +
    {% endwith %} diff --git a/src/templates/robots.txt b/src/templates/robots.txt index 379da7ed..8a05b468 100644 --- a/src/templates/robots.txt +++ b/src/templates/robots.txt @@ -17,6 +17,7 @@ Disallow: /signup/ Disallow: /oauth/ Disallow: /auth/ Disallow: /o/ +Disallow: /form/ Disallow: /coder/*/ratings/ Disallow: /account/*/ratings/ Disallow: /profile/*/ratings/ diff --git a/src/templates/series_filter.html b/src/templates/series_filter.html index 1a6a43ce..bc1a2b8b 100644 --- a/src/templates/series_filter.html +++ b/src/templates/series_filter.html @@ -15,7 +15,7 @@ {% for series in params.series %} - + {% endfor %} @@ -44,6 +44,9 @@ }; }, processResults: function (data, params) { + {% if with_all %}if ((params.page || 1) == 1) { + data.items.unshift({slug: 'all', text: 'Select All'}) + }{% endif %} return { results: data.items.map(function (item) { return {id: item.slug, text: item.text} }), pagination: { diff --git a/src/templates/settings.html b/src/templates/settings.html index 0d2172ba..66bc55ac 100644 --- a/src/templates/settings.html +++ b/src/templates/settings.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load jsonify %} {% load crispy_forms_tags %} {% block ogtitle %}{% block title %}Settings{% endblock %}{% endblock %} @@ -32,10 +31,16 @@ PREFERENCES_URL = "{% url 'coder:settings' 'preferences' %}" LISTS_URL = "{% url 'coder:settings' 'lists' %}" + CHATS_URL = "{% url 'coder:settings' 'chats' %}" + ACCOUNTS_TAB_URL = "{% url 'coder:settings' 'accounts' %}" FILTERS_URL = "{% url 'coder:settings' 'filters' %}" CALENDARS_URL = "{% url 'coder:settings' 'calendars' %}" SUBSCRIPTIONS_URL = "{% url 'coder:settings' 'subscriptions' %}" RESOURCES_URL = "{% url 'clist:resources' %}" + EXTRA_URL_ICON = `{% icon_to 'extra_url' '' %}` + + ACCOUNTS_URL = "{% url 'coder:accounts' %}" + CODERS_URL = "{% url 'coder:coders' %}" SUBSCRIPTIONS_METHODS = [ {% for k, v in notifications %}{% if forloop.counter0 %}, {% endif %}{'id': '{{ k }}', 'text': '{{ v }}'}{% endfor %} ] SHARE_TO_CATEGORY = { 'disable': 'Disable' {% for k, v in notifications %}, '{{ k }}': '{{ v }}'{% endfor %} } @@ -47,6 +52,9 @@ ACCESS_LEVELS = [ {% for access_level in access_levels %}{% if forloop.counter0 %}, {% endif %}{'id': '{{ access_level.value }}', 'text': '{{ access_level.label }}'}{% endfor %} ] WEEK_DAYS = [{'id': '2', 'text': 'Mon'}, {'id': '3', 'text': 'Tue'}, {'id': '4', 'text': 'Wed'}, {'id': '5', 'text': 'Thu'}, {'id': '6', 'text': 'Fri'}, {'id': '7', 'text': 'Sat'}, {'id': '1', 'text': 'Sun'}] + + SUBSCRIPTION_N_LIMIT = {{ coder.subscription_n_limit }} + SUBSCRIPTION_TOP_N_LIMIT = {{ coder.subscription_top_n_limit }} @@ -63,11 +71,9 @@ Filters Notifications Lists - {% if my_chats %} Chats - {% endif %} Calendars - {% comment %} Subscriptions {% endcomment %} + Subscriptions
    @@ -180,10 +186,10 @@ {% endifchanged %} {% endfor %}
    @@ -204,37 +210,17 @@ {% if my_list.access_level == 'restricted' %} and {{ my_list.shared_with|length }} shared with coder(s){% endif %} - +

    {% endfor %}

    - {% if my_chats %}
    -
    - - {% for chat in my_chats %} - - {% for field in my_chats_fields %} - - {% endfor %} - {% if perms.chat.change_chat %} - - {% endif %} - - {% endfor %} -
    - {% with value=chat|get_item:field %} - {% if value is None %}—{% else %}{{ value }}{% endif %} - {% endwith %} - - -
    -
    + {% include "settings_chats.html" with title="Joined" chats=chats.joined fields=chats.fields %} + {% if chats.owned %}{% include "settings_chats.html" with title="Owned" chats=chats.owned fields=chats.fields %}{% endif %}
    - {% endif %}
    {% if not notification_form.errors %} @@ -267,8 +253,8 @@ {% if n.last_time %}check in {{ n.last_time|timezone:coder.timezone|format_time:"%d.%m %a %H:%M" }}{% else %}nothing{% endif %} - - + + {% if perms.notification.change_notification %} {% endif %} @@ -334,6 +320,11 @@
    {% if token %}{{ token.user_id }}{% if token.email %} ({{ token.email }}){% endif %}{% else %}connect{% endif %}
     + {% if token and token.expires_at and service.refresh_token_uri %} + +
    {% icon_to 'expires' '' %}
    +     + {% endif %} {% if token and tokens|length > 1 %} {% endif %} @@ -356,13 +347,25 @@
    - {% for account in coder.account_set_order_by_pk reversed %} -
    @@ -395,7 +398,7 @@ - +
    {% endfor %} @@ -404,8 +407,9 @@
    Create + {{ subscriptions|length }} of {{ coder.n_subscriptions_limit }} {% if perms.notification.view_subscription %} - + {% endif %}
    diff --git a/src/templates/settings_chats.html b/src/templates/settings_chats.html new file mode 100644 index 00000000..94ce95a4 --- /dev/null +++ b/src/templates/settings_chats.html @@ -0,0 +1,30 @@ +

    {{ title }}

    +{% if not chats %} +
    Empty list
    +{% else %} +
    + + + {% for field in fields %} + + {% endfor %} + +{% for chat in chats %} + + {% for field in fields %} + + {% endfor %} + {% if perms.chat.change_chat %} + + {% endif %} + +{% endfor %} +
    {{ field|title_field }}
    + {% with value=chat|get_item:field %} + {% if value is None %}—{% else %}{{ value }}{% endif %} + {% endwith %} + + +
    +
    +{% endif %} diff --git a/src/templates/settings_subscription.html b/src/templates/settings_subscription.html index f6d91a93..dfd1d6d1 100644 --- a/src/templates/settings_subscription.html +++ b/src/templates/settings_subscription.html @@ -1,5 +1,5 @@ -
    -
    +
    +
    @@ -29,34 +29,76 @@ {% endif %} - {% if subscription.account %} + {% if subscription.with_first_accepted %} - - + + + + {% endif %} + {% if subscription.top_n %} + + + {% endif %} - {% if subscription.coder_list %} + {% if subscription.coder_list_id %} {% endif %} - {% if subscription.coder_chat %} + {% if subscription.coder_chat_id %} - + + + {% endif %} + {% if subscription.accounts.all %} + + {% with n_split=3 %} + + + {% endwith %} + + {% endif %} + {% if subscription.coders.all %} + + {% with n_split=3 %} + + + {% endwith %} {% endif %} diff --git a/src/templates/standings.html b/src/templates/standings.html index cc70ce1c..5d245526 100644 --- a/src/templates/standings.html +++ b/src/templates/standings.html @@ -223,6 +223,7 @@

    updated + {% if contest.parsed_percentage and contest.parsed_percentage < 100 %}{{ contest.parsed_percentage|floatformat:1 }}%{% endif %} {{ contest.parsed_time|timezone:timezone|naturaltime }}
    diff --git a/src/templates/standings_account.html b/src/templates/standings_account.html index bdd7c1bc..929cde47 100644 --- a/src/templates/standings_account.html +++ b/src/templates/standings_account.html @@ -15,22 +15,18 @@ {% define country.name as account_country_name %} {% define country.code as account_country_code %} {% define country.flag_code as account_country_flag %} - {% else %} + {% endif %} + {% if not account_country_code %} {% define statistic.addition.country|get_country_code as code %} + {% if not code and members %} + {% get_country_from_members preload_statistics_data.accounts members as code %} + {% endif %} {% if code %} {% define code as account_country_code %} - {% define statistic.addition.country as account_country_name %} + {% define code|get_country_name as account_country_name %} {% define code as account_country_flag %} - {% elif members %} - {% get_country_from_members preload_statistics_data.accounts members as code %} - {% if code %} - {% define code as account_country_code %} - {% define code|get_country_name as account_country_name %} - {% define code as account_country_flag %} - {% endif %} {% endif %} {% endif %} - {% if account_country_code %} {% endif %} @@ -43,14 +39,19 @@
    {% if account.url and not team_id %}{% elif '_account_url' in statistic.addition %}{% endif %} - {% if with_detail and resource.info.standings.subname and account.key|split_account_key:resource.info.standings.subname %} - {% with names=account.key|split_account_key:resource.info.standings.subname %} - {{ names.0 }}
    {{ names.1 }}
     - {% endwith %} + {% if account.deleted %} + {% define "strike" as account_tag %} + {% else %} + {% define "span" as account_tag %} + {% endif %} + {% define False as subnames %} + {% if resource.info.standings.subname and account.key|split_account_key:resource.info.standings.subname %} + {% define account.key|split_account_key:resource.info.standings.subname as subnames %} + <{{ account_tag }}>{% trim_to subnames.0 40 %} {% elif statistic.addition.name and account.key|has_season:statistic.addition.name or members or statistic.addition.name and resource.info.standings.name_instead_key or statistic.addition.name and statistic.addition|get_item:"_name_instead_key" %} - {% trim_to statistic.addition.name 50 %} + <{{ account_tag }}>{% trim_to statistic.addition.name 50 %} {% else %} - {{ account.key }} + <{{ account_tag }}{% if statistic.addition.name or account.name %} title="{{ statistic.addition.name|default:account.name }}" data-placement="top"{% endif %} data-toggle="tooltip">{{ account.key }} {% endif %} {% if statistic.virtual_start %}Your virtual participation{% endif %} @@ -58,7 +59,7 @@ {% if account.url and not team_id or '_account_url' in statistic.addition %}    {% endif %} {% with addition_countries=statistic.addition|get_item:"_countries" %} - {% if addition_countries %} + {% if not members and addition_countries %} {% for country_name in addition_countries %} {% with country_code=country_name|get_country_code %} {% if country_code %} @@ -72,7 +73,9 @@ {% endwith %}
    - {% if not with_detail %}{% include "standings_account_members.html" with members=members except_country_code=account_country_code inline=True %}{% endif %} + {% if not with_detail %} + {% include "standings_account_members.html" with subnames=subnames members=members except_country_code=account_country_code inline=True %} + {% endif %} {% if not without_addition_url %} @@ -116,7 +119,9 @@ {% endif %} - {% if with_detail %}{% include "standings_account_members.html" with members=members except_country_code=account_country_code inline=False %}{% endif %} + {% if with_detail %} + {% include "standings_account_members.html" with subnames=subnames members=members except_country_code=account_country_code inline=False %} + {% endif %} {% endwith %} diff --git a/src/templates/standings_account_members.html b/src/templates/standings_account_members.html index 0957461d..24525555 100644 --- a/src/templates/standings_account_members.html +++ b/src/templates/standings_account_members.html @@ -1,3 +1,6 @@ +{% if subnames %} +
    {{ subnames.1 }}
    +{% endif %} {% if members %}
    {% if inline %}:{% endif %} {% if with_fixed_width %} @@ -8,11 +11,11 @@ {% for member in members %} {% if member and member.account %} - {% with account=preload_statistics_data.accounts|get_item:member.account %} + {% with account=preload_statistics_data.accounts|get_item:member.account without_country=member|get_item:'without_country' %} {% if not account %} {{ member.account }} {% else %} - {% include "account_table_cell.html" with account=account addition=None resource=account.resource without_circle=True with_fixed_width=False without_inline_url=True %} + {% include "account_table_cell.html" with account=account addition=None resource=account.resource without_avatar=account.resource|ne:resource without_circle=True with_fixed_width=False without_inline_url=True without_country=has_country|iffalse:without_country %} {% endif %} {% endwith %} {% elif member and member.coder %} diff --git a/src/templates/standings_list_filters.html b/src/templates/standings_list_filters.html index 9e1c9deb..605d4e4a 100644 --- a/src/templates/standings_list_filters.html +++ b/src/templates/standings_list_filters.html @@ -39,7 +39,7 @@ {% if favorite_contests %}{% include 'favorite_filter.html' %}{% endif %} - {% include 'series_filter.html' %} + {% include 'series_filter.html' with with_all=True %} {% if request.GET.with_submissions|is_yes %} diff --git a/src/templates/standings_problem_progress.html b/src/templates/standings_problem_progress.html index c7257b43..e110e994 100644 --- a/src/templates/standings_problem_progress.html +++ b/src/templates/standings_problem_progress.html @@ -12,7 +12,7 @@
    -
    +
    {% else %}
    diff --git a/src/tg/bot.py b/src/tg/bot.py index 622985e0..dc638fed 100644 --- a/src/tg/bot.py +++ b/src/tg/bot.py @@ -12,15 +12,19 @@ import pytz import telegram from django.conf import settings -from django.db.models import Q +from django.db.models import Prefetch, Q from django.urls import reverse from django.utils.timezone import now from pytimeparse.timeparse import timeparse +from sql_util.utils import SubqueryCount from telegram.constants import MAX_MESSAGE_LENGTH from clist.api.v2 import ContestResource from clist.models import Contest, Resource -from clist.templatetags.extras import as_number, hr_timedelta, md_escape +from clist.templatetags.extras import as_number, hr_timedelta, md_escape, md_url, md_url_text +from notification.models import Subscription +from notification.utils import compose_message_by_problems +from ranking.models import ParseStatistics, Statistics from tg.models import Chat, History logging.basicConfig(level=logging.DEBUG) @@ -58,19 +62,6 @@ def escape(*args): return tuple(map(md_escape, args)) -def fix_url_text(msg): - - def url_text_repl(entry): - text = entry.group(1) - url = entry.group(2) - text = re.sub(r'\\', '', text) - text = re.sub(r'\[', '((', text) - text = re.sub(r'\]', '))', text) - return f'[{text}]({url})' - - return re.sub(r'\[([\]]*?)\]\(([\)]*?)\)', url_text_repl, msg) - - class Bot(telegram.Bot): ADMIN_CHAT_ID = settings.TELEGRAM_ADMIN_CHAT_ID @@ -116,7 +107,7 @@ def update_chat_info(self, chat, message): title = '@' + str(message_from[k]) if title != chat.title: chat.title = title - chat.save() + chat.save(update_fields=['title']) break names = [] @@ -124,9 +115,9 @@ def update_chat_info(self, chat, message): if k in message_from: names.append(message_from[k]) name = ' '.join(names).strip() - if name != chat.name: + if name and name != chat.name: chat.name = name - chat.save() + chat.save(update_fields=['name']) def start(self, args): success = False @@ -337,6 +328,209 @@ def unlink(self, args): self.chat.delete() self.chat_ = False + def subscribe(self, args): + + n_aggregated = 0 + aggregated_msg = None + + def messaging(msg): + nonlocal aggregated_msg, n_aggregated + if not aggregated_msg: + n_aggregated = 0 + if n_aggregated == 20 or len(aggregated_msg) + len(msg) + 1 >= MAX_MESSAGE_LENGTH: + yield aggregated_msg + aggregated_msg = '' + n_aggregated = 0 + if aggregated_msg: + aggregated_msg += '\n' + aggregated_msg += msg + n_aggregated += 1 + + def show_subscribed(method): + nonlocal aggregated_msg + subscriptions = Subscription.objects.filter(coder=self.coder, method=method, contest__isnull=False) + subscriptions = subscriptions.select_related('contest') + subscriptions = subscriptions.annotate(n_accounts=SubqueryCount('accounts')) + if subscriptions: + aggregated_msg = '' + yield from messaging('Subscribed contests:') + for subscription in subscriptions: + contest = subscription.contest + message = (f'[{contest.title}]({md_url(contest.actual_url)}) `{contest.pk}` ' + f', {subscription.n_accounts} account(s)' + + (f' + top {subscription.top_n}' if subscription.top_n else '') + + (' + first ac' if subscription.with_first_accepted else '')) + yield from messaging(message) + yield aggregated_msg + + if not self.is_private: + chat = self.group or self.chat + if not chat or chat.coder != self.coder: + yield 'Use /iamadmin before subscribe.' + return + + method = 'telegram' + if not self.coder or not self.is_private: + method += f':{self.chat_id}' + if self.thread_id: + method += f':{self.thread_id}' + + if args.contest: + contest = Contest.get(args.contest) + else: + contest = ParseStatistics.relevant_contest() + if not contest: + yield 'No set contest. Use `-c` or `--contest` option to choose contest.' + yield from show_subscribed(method) + return + + contest_msg = f'[{contest.title}]({md_url(contest.actual_url)}) `{contest.pk}`' + if args.show: + yield f'Current contest: {contest_msg}.' + yield from show_subscribed(method) + return + + subscription, created = Subscription.objects.get_or_create(coder=self.coder, contest=contest, method=method) + if created: + method_chat = f'telegram:{self.chat_id}' + already_subscriptions = Subscription.objects.filter(Q(coder__isnull=False, coder=self.coder) | + Q(method=method_chat) | + Q(method__startswith=f'{method_chat}:')) + n_subscriptions_limit = settings.CODER_N_SUBSCRIPTIONS_LIMIT_ + if already_subscriptions.count() > n_subscriptions_limit: + subscription.delete() + yield (f'You have reached the limit of {n_subscriptions_limit} subscriptions.\n' + f'Remove unnecessary subscriptions with `-r` or `--remove` options.') + return + + if args.list: + if subscription.is_empty(): + subscription.delete() + yield 'No subscribed participants.' + return + accounts = subscription.accounts.all() + statistics_prefetch = Prefetch('statistics_set', + queryset=Statistics.objects.filter(contest=contest), + to_attr='contest_statistics') + accounts = accounts.prefetch_related(statistics_prefetch) + + statistics_filter = Q(account__in=accounts) + if subscription.top_n: + statistics_filter |= Statistics.top_n_filter(subscription.top_n) + if subscription.with_first_accepted: + statistics_filter |= Statistics.first_ac_filter() + contest_statistics = Statistics.objects.filter(contest=contest).filter(statistics_filter) + contest_statistics = contest_statistics.select_related('account', 'contest__resource') + contest_statistics = contest_statistics.order_by('place_as_int', 'pk') + + aggregated_msg = '' + seen_accounts = set() + for stat in contest_statistics: + account_msg = compose_message_by_problems([], stat, {}, contest) + yield from messaging(account_msg) + seen_accounts.add(stat.account_id) + for account in accounts: + if account.pk in seen_accounts: + continue + account_msg = f'[{md_url_text(account.display())}]({md_url(account.url)})' + yield from messaging(account_msg) + yield aggregated_msg + return + + skip_names_info = False + + if args.first_ac is not None: + skip_names_info = True + first_ac_msg = 'subscribed to first accepted' if args.first_ac else 'unsubscribed from first accepted' + if subscription.with_first_accepted != args.first_ac: + subscription.with_first_accepted = args.first_ac + subscription.save(update_fields=['with_first_accepted']) + first_ac_msg = first_ac_msg.capitalize() + else: + first_ac_msg = f'Already {first_ac_msg}' + yield f'{first_ac_msg} for {contest_msg}.' + + if args.top_n is not None: + skip_names_info = True + args.top_n = args.top_n or None + if args.top_n: + if not 1 <= args.top_n <= settings.CODER_SUBSCRIPTION_TOP_N_LIMIT_: + yield rf'Top N should be in range \[1, {settings.CODER_SUBSCRIPTION_TOP_N_LIMIT_}].' + return + top_n_msg = f'subscribed to top {args.top_n}' if args.top_n else 'unsubscribed from top' + if subscription.top_n != args.top_n: + subscription.top_n = args.top_n + subscription.save(update_fields=['top_n']) + top_n_msg = top_n_msg.capitalize() + else: + top_n_msg = f'Already {top_n_msg}' + yield f'{top_n_msg} for {contest_msg}.' + + accounts_pk_set = set(subscription.accounts.values_list('pk', flat=True)) + processed = set() + changed = False + aggregated_msg = '' + subscription_n_limit = settings.CODER_SUBSCRIPTION_N_LIMIT_ + for name in args.names: + qs = Statistics.objects.filter(contest=contest) + qs = qs.filter(Q(account__name__icontains=name) | Q(account__key__icontains=name)) + qs = qs.select_related('account') + exact = any(statistic.account.name == name or statistic.account.key == name for statistic in qs) + for statistic in qs.select_related('account'): + account = statistic.account + if exact and account.name != name and account.key != name: + continue + if account in processed: + continue + processed.add(account) + account_msg = f'[{md_url_text(account.display())}]({md_url(account.url)})' + if args.remove: + if account.pk in accounts_pk_set: + accounts_pk_set.remove(account.pk) + yield from messaging(f'Unsubscribed {account_msg}.') + changed = True + else: + if account.pk not in accounts_pk_set: + if len(accounts_pk_set) >= subscription_n_limit: + yield from messaging(f'You have reached the limit of {subscription_n_limit} participants.') + break + accounts_pk_set.add(account.pk) + yield from messaging(f'Subscribed {account_msg}.') + changed = True + else: + yield from messaging(f'Already subscribed {account_msg}.') + yield aggregated_msg + + if changed: + subscription.accounts.set(accounts_pk_set) + if subscription.is_empty(): + subscription.delete() + if not args.names and args.remove: + if subscription.pk: + subscription.delete() + yield f'Unsubscribed from all participants of {contest_msg}.' + else: + yield f'No subscribed participants for {contest_msg}.' + elif not contest.n_statistics: + if contest.is_coming(): + yield 'Contest has not started yet, please try again later.' + else: + yield 'Participant list is unavailable, please try again later.' + elif skip_names_info: + return + elif not args.names: + yield f'Set the names of participants for {contest_msg}.' + elif not changed: + yield f'Participants not found to change for {contest_msg}.' + + def unsubscribe(self, args): + for action in self.subscribe_parser._actions: + if action.dest == 'help' or action.dest in args: + continue + setattr(args, action.dest, action.default) + args.remove = True + yield from self.subscribe(args) + @property def parser(self): if not hasattr(self, 'parser_'): @@ -379,6 +573,29 @@ def parser(self): command_p.add_parser('/unlink', description='Unlink account') + subscribe_p = command_p.add_parser('/subscribe', description='Subscribe to participants') + subscribe_p.add_argument('names', metavar='NAME', nargs='*', + help='Participants names (partial matches allowed). ' + 'Double quote names with spaces') + subscribe_p.add_argument('-c', '--contest', help='Contest id, series or name') + subscribe_p.add_argument('-fa', '--first-ac', action='store_true', + help='Subscribe to first accepted', default=None) + subscribe_p.add_argument('-nofa', '--no-first-ac', dest='first_ac', action='store_false', + help='Unsubscribe to first accepted') + subscribe_p.add_argument('-tn', '--top-n', type=int, metavar='N', + help='Subscribe to top N. Set 0 to unsubscribe top') + subscribe_p.add_argument('-r', '--remove', action='store_true', + help='Remove subscription or specified participants if specified') + subscribe_p.add_argument('-l', '--list', action='store_true', help='List subscribed participants') + subscribe_p.add_argument('-s', '--show', action='store_true', help='Show current contest and subscriptions') + self.subscribe_parser = subscribe_p + + unsubscribe_p = command_p.add_parser('/unsubscribe', description='Unsubscribe from participants') + unsubscribe_p.add_argument('names', metavar='NAME', nargs='*', + help='Participants names (partial matches allowed). ' + 'Double quote names with spaces') + unsubscribe_p.add_argument('-c', '--contest', help='Contest id, series or name') + for a in list_p._actions: if not isinstance(a, argparse._HelpAction): if a.default is not None: @@ -403,12 +620,10 @@ def execute_command(self, raw_query): ) regex = '(^%s)@%s' % (regex, settings.TELEGRAM_NAME) query = re.sub(regex, r'\1', query) - args = self.parser.parse_args(shlex.split(query)) - if args.command in ['/prev', '/next', '/repeat']: if not self.chat or not self.chat.last_command: - yield 'Sorry, not found previous command' + yield 'Not found previous command' return c = args.command dargs = vars(args) @@ -432,7 +647,7 @@ def execute_command(self, raw_query): dargs = vars(args) dargs['chat_id__'] = self.chat_id self.chat.last_command = dargs - self.chat.save() + self.chat.save(update_fields=['last_command']) for msg in getattr(self, args.command[1:])(args): paging = getattr(args, 'paging__', False) @@ -455,13 +670,10 @@ def execute_command(self, raw_query): self.delete_message(self.message['chat']['id'], self.message['message_id']) except telegram.error.BadRequest: pass - - except ArgumentCommandError: - pass - except ArgumentParserError as e: - yield 'I\'m sorry, could you clarify:\n' + escape(str(e)) + except (ArgumentParserError, ArgumentCommandError) as e: + yield 'Could you please clarify:\n' + escape(str(e)) except NeedHelpError as e: - yield 'If you need help, please:\n' + escape(str(e)) + yield 'If you need help, please:\n```help\n' + str(e) + '\n```' except Exception as e: self.sendMessage(self.ADMIN_CHAT_ID, 'Query: %s\n\n%s' % (raw_query, format_exc())) yield 'Oops, I\'m having a little trouble:\n' + escape(str(e)) @@ -471,7 +683,6 @@ def send_message(self, msg, chat_id=None, reply_markup=None): msg = {'text': msg} if not msg['text']: return - msg['text'] = fix_url_text(msg['text']) if len(msg['text']) > MAX_MESSAGE_LENGTH: msg['text'] = msg['text'][:MAX_MESSAGE_LENGTH - 3] + '...' @@ -530,6 +741,7 @@ def incoming(self, raw_data): self.chat_id = str(self.message['chat']['id']) self.chat_type = self.message['chat'].get('type') + self.is_private = self.chat_type == 'private' if self.message.get('is_topic_message'): thread = self.message.get('reply_to_message', {}).get('forum_topic_created') @@ -559,15 +771,12 @@ def incoming(self, raw_data): message_id=self.message['message_id'], ) - if not self.coder and was_messaging: - self.send_message(f'Follow {self.follow_url} to connect your account.') - else: - if self.coder and self.coder.settings.get('telegram', {}).get('unauthorized', False): - self.coder.settings.setdefault('telegram', {})['unauthorized'] = False - self.coder.save() - chat = self.group or self.chat - if chat: - History.objects.create(chat=chat, message=data).save() + if self.coder and not was_messaging and self.coder.settings.get('telegram', {}).get('unauthorized', False): + self.coder.settings.setdefault('telegram', {})['unauthorized'] = False + self.coder.save(update_fields=['settings']) + chat = self.group or self.chat + if chat: + History.objects.create(chat=chat, message=data).save() except Exception as e: self.logger.info('Exception incoming message:\n%s\n%s' % (format_exc(), raw_data)) self.logger.error(f'Exception incoming message: {e}') diff --git a/src/tg/models.py b/src/tg/models.py index 9306c472..bf545130 100644 --- a/src/tg/models.py +++ b/src/tg/models.py @@ -1,4 +1,6 @@ from django.db import models +from django.db.models.signals import m2m_changed +from django.dispatch import receiver from pyclist.models import BaseModel from ranking.models import Account @@ -38,6 +40,24 @@ def get_notification_method(self): class Meta: unique_together = ['chat_id', 'thread_id'] + def update_coders_or_accounts(self): + coders, accounts = list(self.coders.all()), list(self.accounts.all()) + for subscription in self.subscription_set.all(): + subscription.coders.set(coders) + subscription.accounts.set(accounts) + + +@receiver(m2m_changed, sender=Chat.coders.through) +@receiver(m2m_changed, sender=Chat.accounts.through) +def update_coders_or_accounts(sender, instance, reverse, pk_set, action, **kwargs): + if not action.startswith('post_'): + return + if reverse: + for chat in Chat.objects.filter(pk__in=pk_set): + chat.update_coders_or_accounts() + else: + instance.update_coders_or_accounts() + class History(BaseModel): LIMIT_BY_CHAT = 7 diff --git a/src/true_coders/admin.py b/src/true_coders/admin.py index d17dfc3b..e3a0ec58 100644 --- a/src/true_coders/admin.py +++ b/src/true_coders/admin.py @@ -11,6 +11,12 @@ class CoderAdmin(BaseModelAdmin): list_display = ['username', 'global_rating', 'last_activity', 'settings'] list_filter = ['party', 'account__resource'] + def get_readonly_fields(self, request, obj=None): + return ( + ['n_accounts', 'n_contests', 'n_subscribers', 'last_activity'] + + super().get_readonly_fields(request, obj) + ) + def clean_settings(self, request, queryset): count = 0 for c in queryset: diff --git a/src/true_coders/management/commands/fill_coder_problems.py b/src/true_coders/management/commands/set_coder_problems.py similarity index 91% rename from src/true_coders/management/commands/fill_coder_problems.py rename to src/true_coders/management/commands/set_coder_problems.py index f5e3bd1a..a78cece6 100644 --- a/src/true_coders/management/commands/fill_coder_problems.py +++ b/src/true_coders/management/commands/set_coder_problems.py @@ -17,11 +17,11 @@ class Command(BaseCommand): - help = 'Fill coder problems using linked accounts' + help = 'Set coder problems using linked accounts' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.logger = getLogger('coders.fill_coder_problems') + self.logger = getLogger('coders.set_coder_problems') def add_arguments(self, parser): parser.add_argument('-r', '--resources', metavar='HOST', nargs='*', help='resources hosts') @@ -40,10 +40,10 @@ def handle(self, *args, **options): args = AttrDict(options) coders = Coder.objects.all() - update_need_fill_coder_problems = False + update_need_set_coder_problems = False if args.no_filled: - coders = coders.filter(Q(settings__need_fill_coder_problems=True)) - update_need_fill_coder_problems = True + coders = coders.filter(Q(settings__need_set_coder_problems=True)) + update_need_set_coder_problems = True if args.coders: coders_filters = Q() @@ -51,7 +51,7 @@ def handle(self, *args, **options): coders_filters |= Q(username=c) coders = coders.filter(coders_filters) self.log_queryset('coders', coders) - update_need_fill_coder_problems = False + update_need_set_coder_problems = False resources = Resource.objects.all() if args.resources: @@ -64,7 +64,7 @@ def handle(self, *args, **options): if not args.coders: coders = coders.annotate(has_resource=Exists('account', filter=Q(account__resource__in=resources))) coders = coders.filter(has_resource=True) - update_need_fill_coder_problems = False + update_need_set_coder_problems = False if args.contest: contest = Contest.objects.get(pk=args.contest) @@ -73,7 +73,7 @@ def handle(self, *args, **options): coders = coders.filter(has_account=True) self.log_queryset('contest problems', problems) self.log_queryset('contest coders', coders) - update_need_fill_coder_problems = False + update_need_set_coder_problems = False else: problems = None @@ -140,8 +140,8 @@ def process_problem(problems, desc): for resource in tqdm(coder_resources, total=len(coder_resources), desc='resources'): resource_problems = resource.problem_set.all() process_problem(resource_problems, desc=f'{resource}') - if update_need_fill_coder_problems: - coder.settings.pop('need_fill_coder_problems', None) + if update_need_set_coder_problems: + coder.settings.pop('need_set_coder_problems', None) coder.save(update_fields=['settings']) self.logger.info(f'n_created = {n_created}, n_deleted = {n_deleted}, n_total = {n_total}') diff --git a/src/true_coders/migrations/0070_coder_n_subscribers.py b/src/true_coders/migrations/0070_coder_n_subscribers.py new file mode 100644 index 00000000..88cb7b9b --- /dev/null +++ b/src/true_coders/migrations/0070_coder_n_subscribers.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1 on 2024-11-17 15:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('true_coders', '0069_listvalue_true_coders_coder_l_1cff61_idx'), + ] + + operations = [ + migrations.AddField( + model_name='coder', + name='n_subscribers', + field=models.IntegerField(blank=True, db_index=True, default=0), + ), + ] diff --git a/src/true_coders/models.py b/src/true_coders/models.py index 48c9e050..913a0a43 100644 --- a/src/true_coders/models.py +++ b/src/true_coders/models.py @@ -10,6 +10,7 @@ from django.core.cache import cache from django.db import models from django.db.models import Case, Count, F, Q, When, signals +from django.db.models.signals import post_delete, post_save from django.dispatch import receiver from django.utils import timezone from django_countries.fields import CountryField @@ -37,6 +38,7 @@ class Coder(BaseModel): addition_fields = models.JSONField(default=dict, blank=True) n_accounts = models.IntegerField(default=0, db_index=True) n_contests = models.IntegerField(default=0, db_index=True) + n_subscribers = models.IntegerField(default=0, db_index=True, blank=True) tshirt_size = models.CharField(max_length=10, default=None, null=True, blank=True) is_virtual = models.BooleanField(default=False, db_index=True) global_rating = models.IntegerField(null=True, blank=True, default=None, db_index=True) @@ -175,6 +177,24 @@ def get_ordered_resources(self): def display_name(self): return self.settings['display_name'] if self.is_virtual else self.username + @property + def detailed_name(self): + if self.is_virtual: + return self.settings['display_name'] + + if self.user.first_name and self.user.last_name: + ret = f'{self.user.first_name} {self.user.last_name}' + elif self.user.first_name or self.user.last_name: + ret = self.user.first_name or self.user.last_name + elif self.first_name_native and self.last_name_native: + ret = f'{self.first_name_native} {self.last_name_native}' + elif self.first_name_native or self.last_name_native: + ret = self.first_name_native or self.last_name_native + else: + return self.username + + return f'{self.username} aka {ret}' + @property def has_global_rating(self): return django_settings.ENABLE_GLOBAL_RATING_ and self.global_rating is not None @@ -189,7 +209,7 @@ def detect_country(self): if self.country != max_country and 2 * max_counter > len(countries): self.country = max_country self.auto_detect_country = True - self.save() + self.save(update_fields=['country', 'auto_detect_country']) def add_account(self, account): coder = self @@ -223,6 +243,27 @@ def primary_accounts(self): qs = qs.distinct('resource') return qs + def get_limit(self, name, default=None): + return self.settings.get('limits', {}).get(name, default) + + def set_limit(self, name, value): + limits = self.settings.setdefault('limits', {}) + ret = limits.get(name) != value + limits[name] = value + return ret + + @property + def n_subscriptions_limit(self): + return self.get_limit('n_subscriptions', django_settings.CODER_N_SUBSCRIPTIONS_LIMIT_) + + @property + def subscription_top_n_limit(self): + return self.get_limit('subscription_top_n', django_settings.CODER_SUBSCRIPTION_TOP_N_LIMIT_) + + @property + def subscription_n_limit(self): + return self.get_limit('subscription_n', django_settings.CODER_SUBSCRIPTION_N_LIMIT_) + class CoderProblem(BaseModel): coder = models.ForeignKey(Coder, on_delete=models.CASCADE, related_name='verdicts') @@ -401,9 +442,8 @@ def filter_for_coder(coder): qs = CoderList.objects condition = Q(access_level=AccessLevel.PUBLIC) if coder is not None: - condition |= Q(owner=coder) qs = qs.annotate(has_shared_with=Exists('shared_with_coders', filter=Q(coder=coder))) - condition |= Q(access_level=AccessLevel.RESTRICTED, has_shared_with=True) + condition |= Q(owner=coder) | Q(access_level=AccessLevel.RESTRICTED, has_shared_with=True) return qs.filter(condition) @staticmethod @@ -430,7 +470,7 @@ def coders_and_accounts_ids(uuids, coder=None, logger=None): if logger: logger.warning(f'Ignore list with uuid = "{uuid}"') continue - for v in coder_list.values.select_related('coder').select_related('account'): + for v in coder_list.related_values: if v.coder: coders.add(v.coder.pk) if v.account: @@ -463,6 +503,16 @@ def coders_filter(uuids, coder=None, logger=None): ret = Q(id=0) return ret + @property + def related_values(self): + return self.values.select_related('coder__user', 'account__resource') + + def update_coders_or_accounts(self): + coders, accounts = CoderList.coders_and_accounts_ids([self.uuid], coder=self.owner) + for subscription in self.subscription_set.all(): + subscription.coders.set(coders) + subscription.accounts.set(accounts) + class ListValue(BaseModel): coder = models.ForeignKey(Coder, null=True, blank=True, on_delete=models.CASCADE) @@ -489,6 +539,11 @@ class Meta: ] +@receiver([post_save, post_delete], sender=ListValue) +def update_list(sender, instance, **kwargs): + instance.coder_list.update_coders_or_accounts() + + class ListProblem(BaseModel): problem = models.ForeignKey(Problem, related_name='lists', on_delete=models.CASCADE) coder_list = models.ForeignKey(CoderList, related_name='problems', on_delete=models.CASCADE) diff --git a/src/true_coders/views.py b/src/true_coders/views.py index d216adfb..06b44fd8 100644 --- a/src/true_coders/views.py +++ b/src/true_coders/views.py @@ -7,6 +7,7 @@ from collections import Counter from datetime import datetime, timedelta +import django_rq import humanize import pytz from django.apps import apps @@ -29,7 +30,6 @@ from django.views.decorators.http import require_http_methods from django_countries import countries from django_ratelimit.core import get_usage -from django_rq import job from el_pagination.decorators import page_template, page_templates from sql_util.utils import Exists, SubqueryCount, SubqueryMax, SubquerySum from tastypie.models import ApiKey @@ -47,6 +47,7 @@ from notes.models import Note from notification.forms import Notification, NotificationForm from notification.models import Calendar, NotificationMessage, Subscription +from notification.utils import compose_message_by_problems, send_messages from pyclist.decorators import context_pagination from pyclist.middleware import RedirectException from ranking.models import (Account, AccountRenaming, AccountVerification, Module, Rating, Statistics, VerifiedAccount, @@ -290,6 +291,7 @@ def coders(request, template='coders.html'): 'noajax': True, 'nogroupby': True, 'nourl': True, + 'icon': 'ghost', } chat_fields = None @@ -918,11 +920,8 @@ def settings(request, tab=None): services = Service.active_objects services = services.annotate(n_tokens=Count('token')).order_by('-n_tokens') - selected_resource = request.GET.get('resource') - selected_account = None - if selected_resource: - selected_resource = Resource.objects.filter(host=selected_resource).first() - selected_account = request.GET.get('account') + selected_resource = request.get_resource() + selected_account = request.GET.get('account') if selected_resource else None categories = coder.get_categories() custom_categories = {c.get_notification_method(): c.title for c in coder.chat_set.filter(is_group=True)} @@ -930,10 +929,17 @@ def settings(request, tab=None): my_lists = coder.my_list_set.annotate(n_records=SubqueryCount('values')) my_lists = my_lists.prefetch_related('shared_with_coders') - my_chats = coder.chat_set.order_by('-modified') - my_chats = my_chats.annotate(n_coders=SubqueryCount('coders')) - my_chats = my_chats.annotate(n_accounts=SubqueryCount('accounts')) - my_chats_fields = ['chat_id', 'title', 'name', 'n_coders', 'n_accounts'] + owned_chats = coder.chat_set.order_by('-modified') + owned_chats = owned_chats.annotate(n_coders=SubqueryCount('coders')) + owned_chats = owned_chats.annotate(n_accounts=SubqueryCount('accounts')) + joined_chats = coder.chats.order_by('-modified') + joined_chats = joined_chats.annotate(n_coders=SubqueryCount('coders')) + joined_chats = joined_chats.annotate(n_accounts=SubqueryCount('accounts')) + chats_fields = ['chat_id', 'title', 'name', 'n_coders', 'n_accounts'] + + subscriptions = coder.subscription_set.order_by('-modified') + subscriptions = subscriptions.prefetch_related('coders__user', 'accounts__resource') + subscriptions = subscriptions.select_related('contest__resource') return render( request, @@ -946,11 +952,14 @@ def settings(request, tab=None): "tokens": {t.service_id: t for t in coder.token_set.all()}, "services": services, "my_lists": my_lists, - "my_chats": my_chats, - "my_chats_fields": my_chats_fields, + "chats": { + "owned": owned_chats, + "joined": joined_chats, + "fields": chats_fields, + }, "categories": categories, "calendars": coder.calendar_set.order_by('-modified'), - "subscriptions": coder.subscription_set.order_by('-modified'), + "subscriptions": subscriptions, "event_description": Calendar.EventDescription, "custom_categories": custom_categories, "coder_notifications": coder.notification_set.order_by('method'), @@ -966,7 +975,7 @@ def settings(request, tab=None): ) -@job +@django_rq.job def call_command_parse_statistics(**kwargs): return call_command('parse_statistic', **kwargs) @@ -1305,46 +1314,155 @@ def change(request): n.save() except Exception: return HttpResponseBadRequest('invalid notification id') - elif name == "add-subscription": - if coder.subscription_set.count() >= 3: - return HttpResponseBadRequest("reached the limit number of subscriptions") - contest = get_object_or_404(Contest, pk=request.POST.get("contest")) - account = get_object_or_404(Account, pk=request.POST.get("account")) + elif name == "add-subscription" or name == "edit-subscription": + n_subscriptions_limit = coder.n_subscriptions_limit + if name == "edit-subscription": + n_subscriptions_limit += 1 + if coder.subscription_set.count() >= n_subscriptions_limit: + return HttpResponseBadRequest(f"reached the limit number of subscriptions ({n_subscriptions_limit})") + + resource_id = request.POST.get("resource") or None + contest_id = request.POST.get("contest") or None + coder_list_id = request.POST.get("coder_list") or None + coder_chat_id = request.POST.get("coder_chat") or None + with_first_accepted = is_yes(request.POST.get("with_first_accepted")) + top_n = as_number(request.POST.get("top_n"), force=True) or None + + accounts_limit = coder.subscription_n_limit + account_ids = request.POST.getlist("accounts[]") + if len(account_ids) > accounts_limit: + return HttpResponseBadRequest(f"reached the limit number of accounts ({accounts_limit})") + accounts = Account.objects.filter(pk__in=account_ids).values_list('pk', flat=True) + if resource_id: + accounts = accounts.filter(resource_id=resource_id) + elif contest_id: + accounts = accounts.filter(statistics__contest_id=contest_id) + else: + with_first_accepted = False + top_n = None + accounts = list(accounts) + + coders_limit = coder.subscription_n_limit + coder_ids = request.POST.getlist("coders[]") + if len(coder_ids) > coders_limit: + return HttpResponseBadRequest(f"reached the limit number of coders ({coders_limit})") + coders = Coder.objects.filter(pk__in=coder_ids).values_list('pk', flat=True) + coders = list(coders) + + if top_n and not 1 <= top_n <= coder.subscription_top_n_limit: + return HttpResponseBadRequest(f"top n should be in [1, {coder.subscription_top_n_limit}]") + + n_choosen = bool(accounts or coders) + bool(coder_list_id) + bool(coder_chat_id) + if n_choosen > 1: + return HttpResponseBadRequest("choose only one of accounts/coders or coder list or coder chat") + n_addition = bool(with_first_accepted or top_n) + if n_choosen + n_addition < 1: + return HttpResponseBadRequest("choose at least one of accounts/coders or coder list or coder chat" + " or first accepted/top n") + if coder_list_id: + coder_list = get_object_or_404(coder.my_list_set, pk=coder_list_id) + coders, accounts = CoderList.coders_and_accounts_ids([coder_list.uuid], coder=coder) + if coder_chat_id: + coder_chat = get_object_or_404(coder.chat_set, pk=coder_chat_id) + coders, accounts = coder_chat.coders.all(), coder_chat.accounts.all() + method = request.POST.get("method") categories = [k for k, v in coder.get_notifications()] if method not in categories: return HttpResponseBadRequest("invalid method value") - sub, created = Subscription.objects.get_or_create( - coder=coder, - method=method, - contest=contest, - account=account, - ) - return HttpResponse("created" if created else "ok") + + if name == "add-subscription": + subscription = Subscription.objects.create( + resource_id=resource_id, + contest_id=contest_id, + with_first_accepted=with_first_accepted, + top_n=top_n, + coder_list_id=coder_list_id, + coder_chat_id=coder_chat_id, + coder=coder, + method=method, + ) + elif name == "edit-subscription": + pk = int(request.POST.get("id")) + subscription = Subscription.objects.get(pk=pk, coder=coder) + subscription.resource_id = resource_id + subscription.contest_id = contest_id + subscription.with_first_accepted = with_first_accepted + subscription.top_n = top_n + subscription.coder_list_id = coder_list_id + subscription.coder_chat_id = coder_chat_id + subscription.method = method + subscription.save() + else: + return HttpResponseBadRequest("invalid name") + + subscription.accounts.set(accounts) + subscription.coders.set(coders) + elif name == "disable-subscription" or name == "enable-subscription": + subscription = get_object_or_404(coder.subscription_set, pk=request.POST.get("id")) + subscription.enable = name == "enable-subscription" + subscription.save(update_fields=['enable']) elif name == "delete-subscription": pk = int(request.POST.get("id", -1)) - sub = Subscription.objects.get(pk=pk, coder=coder) - sub.delete() + Subscription.objects.get(pk=pk, coder=coder).delete() + elif name == "view-subscription": + pk = int(request.POST.get("id", -1)) + subscription = Subscription.objects.get(pk=pk, coder=coder) + + if subscription.contest: + contest = subscription.contest + elif subscription.resource: + contest = subscription.resource.latest_parsed_contest() + else: + contest = None + + sent_statistics = set() + + def view_statistic_by_filter(query): + statistics = Statistics.objects.filter(query, skip_in_stats=False) + if contest: + statistics = statistics.filter(contest=contest).order_by('place_as_int') + else: + statistics = statistics.order_by('-contest__end_time')[:1] + for statistic in statistics: + if statistic.pk in sent_statistics: + continue + sent_statistics.add(statistic.pk) + view_contest = contest or statistic.contest + message = compose_message_by_problems('all', statistic, {}, view_contest) + subscription.send(message=message, contest=view_contest) + + if subscription.top_n: + view_statistic_by_filter(Statistics.top_n_filter(subscription.top_n)) + if subscription.with_first_accepted: + view_statistic_by_filter(Statistics.first_ac_filter()) + for subscription_account in subscription.accounts.all(): + view_statistic_by_filter(Q(account=subscription_account)) + for subscription_coder in subscription.coders.all(): + view_statistic_by_filter(Q(account__coders=subscription_coder)) + + if sent_statistics: + send_messages(coders=[coder.username]) elif name == "first-name": if not value: return HttpResponseBadRequest("empty first name") user.first_name = value - user.save() + user.save(update_fields=['first_name']) elif name == "last-name": if not value: return HttpResponseBadRequest("empty last name") user.last_name = value - user.save() + user.save(update_fields=['last_name']) elif name == "first-name-native": if not value: return HttpResponseBadRequest("empty first name in native language") coder.first_name_native = value - coder.save() + coder.save(update_fields=['first_name_native']) elif name == "last-name-native": if not value: return HttpResponseBadRequest("empty last name in native language") coder.last_name_native = value - coder.save() + coder.save(update_fields=['last_name_native']) elif name == "add-account": try: if "resource" in request.POST: @@ -1658,8 +1776,8 @@ def search(request, **kwargs): ret = [{'id': r.id, 'text': r.host, 'icon': r.icon} for r in qs] elif query == 'contests': qs = Contest.objects.select_related('resource') - if 'regex' in request.GET: - qs = qs.filter(get_iregex_filter(request.GET['regex'], 'title')) + if resource_regex := request.GET.get('regex'): + qs = qs.filter(get_iregex_filter(resource_regex, 'title')) if request.GET.get('has_problems') in django_settings.YES_: qs = qs.filter(info__problems__isnull=False, stage__isnull=True).exclude(info__problems__exact=[]) if request.GET.get('has_submissions') in django_settings.YES_: @@ -1672,9 +1790,12 @@ def search(request, **kwargs): if request.user.is_authenticated: qs = qs.annotate(disabled=VirtualStart.contests_filter(request.user.coder)) qs = qs.filter(start_time__lt=timezone.now(), stage__isnull=True, invisible=False) - resources = [r for r in request.GET.getlist('resources[]') if r] - if resources: + if resources := [r for r in request.GET.getlist('resources[]') if r]: qs = qs.filter(resource__pk__in=resources) + if resource_id := request.GET.get('resource'): + qs = qs.filter(resource_id=resource_id) + if no_stage := request.GET.get('no_stage'): + qs = qs.filter(stage__isnull=is_yes(no_stage)) qs = qs.order_by('-end_time', '-id') qs = qs[(page - 1) * count:page * count] ret = [{'id': r.id, 'text': r.title, 'icon': r.resource.icon, 'disabled': getattr(r, 'disabled', False)} @@ -1771,29 +1892,6 @@ def search(request, **kwargs): qs = [(c, n) for c, n in countries if name in n.lower()] qs = qs[(page - 1) * count:page * count] ret = [{'id': c, 'text': n} for c, n in qs] - elif query == 'account-for-add-subscription': - contest = get_object_or_404(Contest, pk=request.GET.get('contest')) - qs = contest.statistics_set.select_related('account') - search = request.GET.get('search') - if search: - qs = qs.filter(Q(account__key__icontains=search) | Q(account__name__icontains=search)) - qs = qs.order_by('place', '-solving') - qs = qs[(page - 1) * count:page * count] - ret = [ - { - 'id': s.account.id, - 'text': f'{s.account.key}, {s.account.name}' if s.account.name else s.account.key, - } - for s in qs - ] - elif query == 'contest-for-add-subscription': - qs = Contest.objects.filter(n_statistics__gt=0, end_time__gte=timezone.now()) - regex = request.GET.get('regex') - if regex: - qs = qs.filter(title__iregex=verify_regex(regex)) - qs = qs.order_by('-end_time', '-id') - qs = qs[(page - 1) * count:page * count] - ret = [{'id': c.id, 'text': c.title} for c in qs] elif query == 'notpast': qs = Contest.objects.filter(end_time__gte=timezone.now()) regex = request.GET.get('regex') @@ -1898,50 +1996,80 @@ def search(request, **kwargs): f_text = str(getattr(f, field_name)) ret.append({'id': f_id, 'text': f_text}) elif query == 'coders': - qs = Coder.objects.all() + qs = Coder.objects.select_related('user') - if 'regex' in request.GET: - qs = qs.filter(get_iregex_filter(request.GET['regex'], 'username')) + if contest_id := request.GET.get('contest'): + qs = qs.annotate(has_contest=Exists('account', filter=Q(account__statistics__contest_id=contest_id))) + qs = qs.filter(has_contest=True) + if resource_id := request.GET.get('resource'): + qs = qs.annotate(has_resource=Exists('account', filter=Q(account__resource_id=resource_id))) + qs = qs.filter(has_resource=True) - order = ['-n_accounts', 'pk'] + order = [] + if search := request.GET.get('search'): + qs = qs.filter( + Q(username__icontains=search) | + Q(user__first_name__icontains=search) | + Q(user__last_name__icontains=search) + ) + qs = qs.annotate(search_weight=Case( + When(username__iexact=search, then=Value(0)), + When(user__first_name__iexact=search, then=Value(1)), + When(user__last_name__iexact=search, then=Value(1)), + default=Value(2), + output_field=IntegerField() + )) + order.append('search_weight') if request.user.is_authenticated: - qs = qs.annotate(iam=Case( + qs = qs.annotate(my_weight=Case( When(pk=request.user.coder.pk, then=Value(0)), default=Value(1), output_field=IntegerField() )) - order.insert(0, 'iam') - qs = qs.order_by(*order, 'pk') + order.append('my_weight') + qs = qs.order_by(*order, '-n_contests', 'pk') qs = qs[(page - 1) * count:page * count] - ret = [{'id': r.id, 'text': r.username} for r in qs] + ret = [{'id': coder.id, 'text': coder.detailed_name} for coder in qs] elif query == 'accounts': - qs = Account.objects.all() - if request.GET.get('resource'): - qs = qs.filter(resource_id=int(request.GET.get('resource'))) - - with_resource = True - if request.GET.get('contest'): - qs = qs.filter(statistics__contest_id=int(request.GET.get('contest'))) + qs = Account.objects + if contest_id := request.GET.get('contest'): + qs = qs.filter(statistics__contest_id=contest_id) + order = ['statistics__place_as_int', '-statistics__solving'] + with_resource = False + elif resource_id := request.GET.get('resource'): + qs = qs.filter(resource_id=resource_id) + order = ['resource_rank'] with_resource = False - if with_resource: + else: + order = ['-n_contests', 'pk'] + with_resource = True qs = qs.select_related('resource') - order = ['-n_contests', 'pk'] - if 'search' in request.GET: - search = request.GET['search'] + if search := request.GET.get('search'): qs = qs.filter(get_iregex_filter(search, 'key', 'name', suffix='__icontains')) - qs = qs.annotate(match=Case( + qs = qs.annotate(search_match=Case( When(Q(key__iexact=search) | Q(name__iexact=search), then=Value(True)), default=Value(False), output_field=BooleanField(), )) - order.insert(0, '-match') - + order = ['-search_match'] + order qs = qs.order_by(*order, 'pk') qs = qs[(page - 1) * count:page * count] - ret = [{'id': r.id, 'text': r.display(with_resource=with_resource)} for r in qs] + ret = [{'id': account.id, 'text': account.display(with_resource=with_resource)} for account in qs] + elif query == 'coder_lists' and request.user.is_authenticated: + qs = request.user.coder.my_list_set.all() + if search := request.GET.get('search'): + qs = qs.filter(name__icontains=search) + qs = qs[(page - 1) * count:page * count] + ret = [{'id': coder_list.id, 'text': coder_list.name} for coder_list in qs] + elif query == 'coder_chats' and request.user.is_authenticated: + qs = request.user.coder.chats.all() + if search := request.GET.get('search'): + qs = qs.filter(title__icontains=search) + qs = qs[(page - 1) * count:page * count] + ret = [{'id': coder_chat.id, 'text': coder_chat.title} for coder_chat in qs] else: return HttpResponseBadRequest(f'invalid query = {escape(query)}') @@ -2113,6 +2241,8 @@ def party(request, slug, tab='ranking'): place = i + 1 s['place'] = place + admins = party.admins.all() + return render( request, 'party.html', @@ -2125,6 +2255,7 @@ def party(request, slug, tab='ranking'): 'party_contests': party_contests, 'results': results, 'coders': coders, + 'admins': admins, 'tab': 'ranking' if tab is None else tab, }, ) @@ -2165,7 +2296,7 @@ def view_list(request, uuid): qs = qs.prefetch_related('values__coder') coder_list = get_object_or_404(qs, uuid=uuid) - is_owner = coder_list.owner == coder + is_owner = coder and coder_list.owner_id == coder.pk can_modify = is_owner request_post = request.session.pop('view_list_request_post', None) or request.POST @@ -2276,7 +2407,7 @@ def add_account(a, log_value=None): return allowed_redirect(request.path) coder_values = {} - for v in coder_list.values.order_by('group_id').all(): + for v in coder_list.related_values.order_by('group_id').all(): data = coder_values.setdefault(v.group_id, {}) data.setdefault('list_values', []).append(v) diff --git a/src/utils/custom_request.py b/src/utils/custom_request.py index 6e06edde..88f06c29 100644 --- a/src/utils/custom_request.py +++ b/src/utils/custom_request.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 from functools import partial +from typing import Optional from django.contrib import messages +from clist.models import Resource + class RequestLogger: @@ -17,10 +20,15 @@ def __getattr__(self, attr): return ret -def get_filtered_list(request, field, options=None): +def get_resource(self, field='resource') -> Optional[Resource]: + resource = self.GET.get(field) + return Resource.get(resource) + + +def get_filtered_list(self, field, options=None): values = [ value - for value in request.GET.getlist(field) + for value in self.GET.getlist(field) if options is not None and value in options or options is None and value ] if values and isinstance(options, list): @@ -30,10 +38,10 @@ def get_filtered_list(request, field, options=None): return values -def get_filtered_value(request, field, options=None, default_first=None, allow_empty=False): +def get_filtered_value(self, field, options=None, default_first=None, allow_empty=False): if allow_empty and '' not in options: options = options + [''] - ret = get_filtered_list(request, field, options) + ret = self.get_filtered_list(field, options) if ret: return ret[-1] if default_first and options: @@ -41,8 +49,17 @@ def get_filtered_value(request, field, options=None, default_first=None, allow_e return None -def custom_request(request): +def set_canonical(self, url): + url = self.build_absolute_uri(url) + self.canonical_url = url + return self + + +def CustomRequest(request): setattr(request, 'logger', RequestLogger(request)) + setattr(request, 'get_resource', partial(get_resource, request)) setattr(request, 'get_filtered_list', partial(get_filtered_list, request)) setattr(request, 'get_filtered_value', partial(get_filtered_value, request)) + setattr(request, 'canonical_url', None) + setattr(request, 'set_canonical', partial(set_canonical, request)) return request diff --git a/src/utils/db.py b/src/utils/db.py index e77bc983..14eae96a 100644 --- a/src/utils/db.py +++ b/src/utils/db.py @@ -18,7 +18,16 @@ def dictfetchone(cursor): def find_app_by_table(table_name): + # First, try to find a model with a direct mapping to the table name for model in apps.get_models(): if model._meta.db_table == table_name: return model._meta.app_label + + # If no direct model found, check if the table is an automatic through table + for model in apps.get_models(): + for field in model._meta.many_to_many: + if field.remote_field.through._meta.db_table == table_name: + return model._meta.app_label + + # If still not found, return None return None diff --git a/src/utils/requester/__init__.py b/src/utils/requester/__init__.py index 081e0394..8b39acd1 100644 --- a/src/utils/requester/__init__.py +++ b/src/utils/requester/__init__.py @@ -95,8 +95,7 @@ def __getattr__(self, name): return getattr(self.args[0], name, None) -def raise_fail(err): - exc = FailOnGetResponse(err) +def raise_fail(err, exc): if exc.code or exc.url: msg = f'code = {exc.code}, url = `{exc.url}`' if exc.response: @@ -114,7 +113,7 @@ class NoVerifyWord(Exception): class Proxer(): DIVIDER = 3 - LIMIT_TIME = 2.5 + LIMIT_TIME = float(environ.get('PROXY_LIMIT_TIME', 3)) def load_data(self): try: @@ -125,6 +124,12 @@ def load_data(self): self._data.setdefault('proxies', {}) self._data.setdefault('sources', {}) self._data.setdefault('deferred', {}) + + env_proxy = environ.get('REQUESTER_PROXY') + if env_proxy and re.match(r'^[0-9]+\.+[0-9]+\.+[0-9]+\.+[0-9]+:[0-9]+$', env_proxy): + self._data['proxies'] = {} + self.add(env_proxy) + for proxy in self.proxies.values(): self.init_proxy(proxy) @@ -291,11 +296,14 @@ def ok(self, proxy, time_response=None): self.print(f'ok, {time_response} with average {self.time_response()}') self.check_proxy() - def fail(self, proxy): + def fail(self, proxy, force=False): with self.lock: if not self.proxy or proxy != self.proxy_address: return - self.proxy['_state'] //= self.DIVIDER + if force: + self.proxy['_state'] = 0 + else: + self.proxy['_state'] //= self.DIVIDER self.proxy['_total_fail'] += 1 self.update_value('_fail', 1) self.check_proxy() @@ -529,7 +537,7 @@ def __init__(self, caching=None, user_agent=None, headers=None, - file_name_with_proxies=default_filepath_proxies): + proxy_filepath=default_filepath_proxies): if cookie_filename: self.cookie_filename = cookie_filename if caching is not None: @@ -550,7 +558,7 @@ def __init__(self, ] self._init_opener_headers = self.headers self.init_opener() - self.set_proxy(proxy, file_name_with_proxies) + self.set_proxy(proxy, proxy_filepath) atexit.register(self.cleanup) def init_opener(self): @@ -591,6 +599,10 @@ def set_proxy(proxy): if self.proxer: self.proxer.connect(req=self, set_proxy=set_proxy) + def proxy_fail(self, proxy=None, force=False): + if self.proxer: + self.proxer.fail(proxy=proxy or self.proxy, force=force) + def get( self, url, @@ -742,14 +754,14 @@ def get( else: self.print(f'[error] code = {error_code}, response = {str(err)[:200]}') self.error = err - if self.proxer: - self.proxer.fail(proxy=str(proxy)) + self.proxy_fail(proxy) + error_exception = FailOnGetResponse(err) if additional_attempts and error_code in additional_attempts: additional_attempt = additional_attempts[error_code] if ( additional_attempt['count'] > 0 and - ('func' not in additional_attempt or additional_attempt['func'](FailOnGetResponse(err))) + ('func' not in additional_attempt or additional_attempt['func'](error_exception)) ): additional_attempt['count'] -= 1 attempt -= 1 @@ -760,8 +772,11 @@ def get( if attempt < n_attempts: sleep(attempt_delay) continue + if (fp := os.environ.get('REQUESTER_PAGE_ON_FAIL')) and error_exception.response: + with open(fp, 'w') as fo: + fo.write(error_exception.response) if self.assert_on_fail: - raise_fail(err) + raise_fail(err, error_exception) else: traceback.print_exc() return @@ -838,7 +853,7 @@ def get( if not return_url and not return_code: return page ret = [page] - if return_url or return_code: + if return_url: ret += [last_url] if return_code: ret += [response.code] @@ -849,7 +864,10 @@ def current_url(self): return self.last_url def head(self, url): - return self.opener.open(url).getheaders() + try: + self.opener.open(url).getheaders() + except urllib.error.HTTPError as e: + raise FailOnGetResponse(e) def geturl(self, url): try: @@ -1030,20 +1048,27 @@ def close(self): self.proxer.save_data() self.save_cookie() + def duplicate(self, **kwargs): + ret = copy.copy(self) + for k, v in kwargs.items(): + setattr(ret, k, v) + ret.init_opener() + return ret + def with_proxy(self, inplace=True, attributes=None, **kwargs): self.save_cookie() - if inplace: - ret = self - else: - ret = copy.copy(self) - ret.init_opener() - ret.set_proxy(proxy=True, **kwargs) + ret = self if inplace else copy.copy(self) + if attributes: orig_attributes = {} for field, value in attributes.items(): orig_attributes[field] = getattr(ret, field, None) setattr(ret, field, value) setattr(ret, 'orig_attributes', orig_attributes) + + if not inplace: + ret.init_opener() + ret.set_proxy(proxy=True, **kwargs) return ret def __enter__(self): diff --git a/src/utils/strings.py b/src/utils/strings.py index 0f5bff9d..352a310e 100644 --- a/src/utils/strings.py +++ b/src/utils/strings.py @@ -1,7 +1,11 @@ import random +import secrets import string from collections import Counter +from bs4 import BeautifulSoup +from markdown import markdown + def trim_on_newline(text, max_length): if len(text) <= max_length: @@ -28,5 +32,25 @@ def list_string_iou(a, b): return intersection / union if union > 0 else 0.0 +def slug_string_iou(a, b): + return list_string_iou(a.split('-'), b.split('-')) + + def random_string(length=40): return ''.join(random.choices(list(string.ascii_letters + string.digits), k=length)) + + +def generate_secret(length=16): + return secrets.token_hex(length) + + +def generate_secret_64(): + return generate_secret(32) + + +def markdown_to_text(markdown_text): + return BeautifulSoup(markdown(markdown_text), 'html.parser').get_text() + + +def markdown_to_html(markdown_text): + return markdown(markdown_text) diff --git a/src/utils/urlutils.py b/src/utils/urlutils.py new file mode 100644 index 00000000..a58502e2 --- /dev/null +++ b/src/utils/urlutils.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + + +from urllib.parse import urljoin + +from django.conf import settings + + +def absolute_url(url): + return urljoin(settings.HTTPS_HOST_URL_, url)

    Method
    Account - {% include "account_table_cell.html" with account=subscription.account resource=subscription.account.resource without_country=True without_avatar=True without_circle=True %} - First ACOn
    Top N{{ subscription.top_n }}
    List - - {{ subscription.coder_list.name }} - + {{ subscription.coder_list.name }} + {% if perms.true_coders.change_coderlist %}{% admin_url subscription.coder_list %}{% endif %}
    Chat{{ subscription.coder_chat.title }} + {{ subscription.coder_chat.title }} + {% if perms.tg.change_chat %}{% admin_url subscription.coder_chat %}{% endif %} +
    Accounts + {% for account in subscription.accounts.all %} + {% if forloop.counter0 == n_split %}{{ n_more }} more{% endif %} + {% endwith %} +
    Coders + {% for coder in subscription.coders.all %} + {% if forloop.counter0 == n_split %}{{ n_more }} more{% endif %} + {% endwith %} +
    - + + + + +