User:DYKHousekeepingBot/Code

dykNomStatsBot.py:

import os
import pathlib
import pywikibot
import sys
import traceback
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from dykNomStatsEngine import DYKNomStatsEngine


class DYKNomStatsBot():
    MINUTES_BETWEEN_UPDATES = 30
    NOM_STATS_UPDATES_PER_DAY = 1440 / MINUTES_BETWEEN_UPDATES
    NOM_STATS_TABLE_LOC = 'Wikipedia:Did you know/DYK hook count'
    # NOM_STATS_TABLE_LOC = 'User:Shubinator/Sandbox/DYK hook count'
    once_daily_modulus = -1

    def __init__(self) -> None:
        pass

    def run(self) -> None:
        self._log('PID: {0}'.format(os.getpid()))

        while self._is_on():
            self._log(datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S %Z'))
            try:
                self.update_nom_stats()
            except:
                self._log('Nom stats exception occurred...')
                traceback.print_exc(file=sys.stdout)

            self.once_daily_modulus = (self.once_daily_modulus + 1) % self.NOM_STATS_UPDATES_PER_DAY
            if self.once_daily_modulus == 0:
                try:
                    self.find_alert_untranscluded_noms()
                except:
                    self._log('Untranscluded nom exception occurred...')
                    traceback.print_exc(file=sys.stdout)

            pywikibot.sleep(self.MINUTES_BETWEEN_UPDATES * 60)

    # ---------------------------------------------
    # Specific to creating the DYK hook count table
    # ---------------------------------------------
    def update_nom_stats(self) -> None:
        html_ttdyk = self._fetch_html_for_wikipage('Template talk:Did you know')
        html_approved = self._fetch_html_for_wikipage('Template talk:Did you know/Approved')
        str_nom_stats_table = DYKNomStatsEngine.parse_and_create_nom_stats_table(html_ttdyk, html_approved, datetime.now(timezone.utc))
        str_table_wo_timestamp = str_nom_stats_table[:str_nom_stats_table.find('Last updated')]
        wikipage_nom_stats = pywikibot.Page(pywikibot.Site(), self.NOM_STATS_TABLE_LOC)
        if str_table_wo_timestamp in wikipage_nom_stats.text:
            self._log('No change since last update')
        else:
            wikipage_nom_stats.text = str_nom_stats_table
            self._edit(wikipage_nom_stats, 'Bot updating DYK nominated hook stats')

    def _fetch_html_for_wikipage(self, str_wikipage_title):
        wikipage = pywikibot.Page(pywikibot.Site(), str_wikipage_title)
        return wikipage.get_parsed_page()  # uses API.php's action=parse

    # ---------------------------------------------
    # Specific to untranscluded noms
    # ---------------------------------------------
    def find_alert_untranscluded_noms(self):
        self._log('Running untranscluded noms check')
        set_untranscluded = self.find_untranscluded_noms()
        self._alert_untranscluded_noms(set_untranscluded)

    def find_untranscluded_noms(self):
        set_pending = self._get_pending_noms()
        self._enumerate_transcluded_pages_and_trim(set_pending, 'Template talk:Did you know')
        self._enumerate_transcluded_pages_and_trim(set_pending, 'Template talk:Did you know/Approved')
        self._enumerate_transcluded_pages_and_trim(set_pending, 'Wikipedia:April Fools\' Main Page/Did you know')
        return set_pending

    def _get_pending_noms(self):
        cat_pending = pywikibot.Category(pywikibot.Site(), 'Category:Pending DYK nominations')
        date_freshest_stale_nom = pywikibot.Site().server_time() - timedelta(days=1)
        pages_pending_aged = cat_pending.articles(sortby='timestamp', endtime=date_freshest_stale_nom)
        set_pending = set()
        for page_pending in pages_pending_aged:
            if page_pending.title().startswith('Template:Did you know nominations/'):
                set_pending.add(page_pending.title())
        return set_pending

    def _enumerate_transcluded_pages_and_trim(self, set_pending, str_transcluded_on):
        page_transcluded_on = pywikibot.Page(pywikibot.Site(), str_transcluded_on)
        for page_transcluded in page_transcluded_on.templates():
            set_pending.discard(page_transcluded.title())

    def _alert_untranscluded_noms(self, set_untranscluded):
        map_nominator_to_noms = defaultdict(list)
        for str_untranscluded in set_untranscluded:
            str_nominator = pywikibot.Page(pywikibot.Site(), str_untranscluded).oldest_revision.user
            if not (self._has_been_alerted(str_untranscluded, str_nominator) or self._already_closed(str_untranscluded)):
                map_nominator_to_noms[str_nominator].append(str_untranscluded)

        for str_nominator, rgstr_untranscluded in map_nominator_to_noms.items():
            self._post_untranscluded_alert(str_nominator, rgstr_untranscluded)

    def _has_been_alerted(self, str_nom_page_title, str_nominator):
        page_nominator_talk = pywikibot.Page(pywikibot.Site(), 'User talk:' + str_nominator)
        has_been_alerted = False
        if page_nominator_talk.exists():
            pywikibot.Site().loadrevisions(page_nominator_talk, user=pywikibot.Site().username())
            has_been_alerted = any(str_nom_page_title in rev.comment for rev in page_nominator_talk._revisions.values())
        # if has_been_alerted:
        #     self._log('User talk:' + str_nominator + ' has been alerted about ' + str_nom_page_title)
        return has_been_alerted

    def _already_closed(self, str_nom_page_title):
        page_nomination = pywikibot.Page(pywikibot.Site(), str_nom_page_title)
        already_closed = 'The result was:' in page_nomination.text
        if already_closed:
            self._log(page_nomination + ' was not properly closed')
        return already_closed

    def _post_untranscluded_alert(self, str_nominator, rgstr_untranscluded) -> None:
        str_message, str_edit_summary = self._construct_talk_page_edit(rgstr_untranscluded)
        self._append_and_edit('User talk:' + str_nominator, str_message, str_edit_summary)

    def _construct_talk_page_edit(self, rgstr_untranscluded):
        str_nom_word = 'nomination' if len(rgstr_untranscluded) == 1 else 'nominations'
        str_talk_page_message = '==Incomplete DYK ' + str_nom_word + '==\n'
        str_talk_page_message += '\n\n'.join(
            '{{{{subst:DYK nomination needs transcluded|{0}}}}}'.format(untranscluded) for untranscluded in rgstr_untranscluded)
        str_talk_page_edit_summary = 'Untranscluded DYK ' + str_nom_word + ' at '
        str_talk_page_edit_summary += ', '.join('[[{0}]]'.format(untranscluded) for untranscluded in rgstr_untranscluded)
        return str_talk_page_message, str_talk_page_edit_summary

    # Edge cases we're handling:
    # * {{nobots}}
    # * Redirects
    # * Page doesn't exist
    # * Edit conflicts
    # * Protected page
    def _append_and_edit(self, str_title, str_message, str_edit_summary) -> None:
        page_to_edit = pywikibot.Page(pywikibot.Site(), str_title)
        if page_to_edit.isRedirectPage():
            page_to_edit = page_to_edit.getRedirectTarget()
        if not page_to_edit.botMayEdit():
            # Attempting to save the page when botMayEdit() is False will throw an OtherPageSaveError
            self._log('Couldn\'t edit ' + page_to_edit.title() + ' due to {{bots}} or {{nobots}}')
            return

        retry = True
        while retry:
            retry = False
            try:
                if page_to_edit.text != '':
                    page_to_edit.text += '\n\n'
                page_to_edit.text += str_message
                self._edit(page_to_edit, str_edit_summary)
            except pywikibot.exceptions.EditConflictError:
                retry = True
                self._log('Edit conflicted on ' + page_to_edit.title() + ' will retry after a short nap')
                pywikibot.sleep(10)  # sleep for 10 seconds
                page_to_edit = pywikibot.Page(pywikibot.Site(), page_to_edit.title())

    # ---------------------------------------------
    # Utility / core
    # ---------------------------------------------
    def _is_on(self):
        wikipage_switch = pywikibot.Page(pywikibot.Site(), 'User:DYKHousekeepingBot/Switch')
        is_wiki_switch_on = wikipage_switch.text.strip().lower() == 'on'
        if not is_wiki_switch_on:
            self._log('Wiki switch is not "on", exiting...')

        with open(str(pathlib.Path(__file__).parent / 'NomStatsBotSwitch.txt'), 'r', encoding='utf-8') as f:
            str_file_switch = f.read()
        is_file_switch_on = str_file_switch.strip().lower() == 'on'
        if not is_file_switch_on:
            self._log('Text file switch is not "on", exiting...')

        return is_wiki_switch_on and is_file_switch_on

    def _edit(self, page_to_edit, str_edit_summary):
        self._log('Editing ' + page_to_edit.title())
        try:
            page_to_edit.save(str_edit_summary, minor=False)
        except pywikibot.exceptions.LockedPageError:  # covers directly protected, cascade protected, salted
            self._log(page_to_edit.title() + ' is protected, skipping...')

    def _log(self, str):
        print(str, flush=True)


def main() -> None:
    bot = DYKNomStatsBot()
    bot.run()

if __name__ == '__main__':
    main()

dykNomStatsEngine.py:

from datetime import datetime, timezone

# DYKHousekeepingBot nom task's parse, logic, and format code
# Do not add Wikipedia read/write code (or any network calls) here as
# this would cause the unit tests to go over the network (no bueno)


class DYKNomStatsEngine():
    @staticmethod
    def parse_and_create_nom_stats_table(html_ttdyk, html_approved, date_now):
        noms_from_ttdyk = DYKNomStatsEngine._parse_page_with_nominations(html_ttdyk, date_now)
        noms_from_approved = DYKNomStatsEngine._parse_page_with_nominations(html_approved, date_now)
        noms_merged = DYKNomStatsEngine._merge_noms_data(noms_from_ttdyk, noms_from_approved)
        return DYKNomStatsEngine._create_nom_stats_table(noms_merged, date_now)

    @staticmethod
    def _parse_page_with_nominations(html_noms_page, date_now):
        # Splice out the special occasion holding area
        idx_special_occasion_start = html_noms_page.find('span class="mw-headline" id="Special_occasion_holding_area"')
        if idx_special_occasion_start > 0:
            idx_special_occasion_end = html_noms_page.find('<h2', idx_special_occasion_start)
            if idx_special_occasion_end < 0:
                idx_special_occasion_end = len(html_noms_page)
            html_noms_page = html_noms_page.replace(html_noms_page[idx_special_occasion_start:idx_special_occasion_end], '')

        str_daily_anchor = 'id="Articles_created/expanded_on'
        rghtml_date_sections = html_noms_page.split(str_daily_anchor)
        del rghtml_date_sections[0]  # splice out table of contents and other gunk
        noms_section_data = dict()
        for html_noms_for_one_day in rghtml_date_sections:
            created_expanded = 'created/expanded on '
            str_section_date = html_noms_for_one_day[html_noms_for_one_day.find(created_expanded) + len(created_expanded):]
            str_section_date = str_section_date[:str_section_date.find('<')]
            num_hooks, num_approved = DYKNomStatsEngine._count_noms_in_section(html_noms_for_one_day)
            num_days_ago = DYKNomStatsEngine._calculate_num_days_ago(str_section_date, date_now)
            noms_section_data[num_days_ago] = NomsSectionData(str_section_date, num_days_ago, num_hooks, num_approved)
        return noms_section_data

    @staticmethod
    def _count_noms_in_section(html_noms_for_one_day):
        rghtml_noms_for_one_day = html_noms_for_one_day.split('<h4')
        num_hooks = 0
        num_approved = 0
        num_archived = 0
        for html_nom in rghtml_noms_for_one_day:
            if '"<div style="display:none">Archived nomination</div>"' in html_nom:
                num_archived += 1
                continue
            idx_approved = max(html_nom.rfind('Symbol_confirmed.svg'),
                               html_nom.rfind('Symbol_voting_keep.svg'))
            idx_declined = max(html_nom.rfind('Symbol_question.svg'),
                               html_nom.rfind('Symbol_redirect_vote_4.svg'),
                               html_nom.rfind('Symbol_possible_vote.svg'),
                               html_nom.rfind('Symbol_delete_vote.svg'))
            if idx_approved > idx_declined:
                num_approved += 1
        num_hooks = len(rghtml_noms_for_one_day) - 1 - num_archived
        return num_hooks, num_approved

    @staticmethod
    def _calculate_num_days_ago(str_date, date_now):
        date_from_section = datetime.strptime(str_date + ' ' + str(date_now.year), '%B %d %Y')
        date_from_section = date_from_section.replace(tzinfo=timezone.utc)
        if date_from_section > date_now:
            date_from_section = date_from_section.replace(year=date_from_section.year - 1)
        return (date_now - date_from_section).days

    @staticmethod
    def _merge_noms_data(dict1, dict2):
        for key in dict2:
            if key in dict1:
                dict1[key] += dict2[key]
            else:
                dict1[key] = dict2[key]
        return dict1

    @staticmethod
    def _create_nom_stats_table(noms_section_data, date_now):
        num_total_hooks = 0
        num_total_approved = 0
        table_lines = [
            '{| class="wikitable" style="text-align:center"',
            '| colspan="3" height="45px" | <big>\'\'\'Count of DYK Hooks\'\'\'</big>',
            '|-',
            '! Section !! # of Hooks !! # Verified',
            ]
        for day_data in sorted(noms_section_data.values()):
            num_total_hooks += day_data.num_hooks
            num_total_approved += day_data.num_approved

            # Color the row
            str_row_color = ' style="background:#ffaaaa"' if day_data.num_days_ago > 7 else ''
            table_lines.append('|-' + str_row_color)

            # Meat
            str_num_hooks = str(day_data.num_hooks) if day_data.num_hooks > 0 else ''
            str_num_approved = str(day_data.num_approved) if day_data.num_approved > 0 else ''
            table_lines.append('| [[{{{{#ifeq:{{{{FULLPAGENAME}}}}|Template talk:Did you know'
                               '||{{{{#ifeq:{{{{FULLPAGENAME}}}}|Template talk:Did you know/Approved'
                               '||Template talk:Did you know}}}}}}}}#Articles created/expanded on '
                               '{0}|{0}]] || {1} || {2}'.format(day_data.str_month_day, str_num_hooks, str_num_approved))

        table_lines.append('|-')
        table_lines.append('! Total !! {0} !! {1}'.format(num_total_hooks, num_total_approved))
        table_lines.append('|-')
        table_lines.append('| colspan=3 align=left|<small>Last updated '
                           '{d:%H}:{d:%M}, {d.day} {d:%B} {d.year} [[Coordinated Universal Time|UTC]]<br>'
                           'Current time is {{{{time}}}}</small>'.format(d=date_now))
        table_lines.append('|}')
        return '\n'.join(table_lines)


class NomsSectionData():
    def __init__(self, str_month_day, num_days_ago, num_hooks, num_approved) -> None:
        self.str_month_day = str_month_day
        self.num_days_ago = num_days_ago
        self.num_hooks = num_hooks
        self.num_approved = num_approved

    def __lt__(self, other):
        return self.num_days_ago > other.num_days_ago

    def __iadd__(self, other):
        self.num_hooks += other.num_hooks
        self.num_approved += other.num_approved
        return self