User:Tony Sidaway/scripts/search

Iron Chicken script:

(define (page-contains-pattern wiki page pattern)
  (and-let* (
    [page-contents (wiki-page-contents wiki page)])
    (string-search pattern page-contents)))

(define (wiki-section-every n lst . opt)
  (let ([label (if (null? opt)
		 (string-append "Section of " (->string n) " number")
		 (car opt))])
    (let loop ([i 0] [lst lst] [out-lst '()])
      (cond
	[(null? lst) out-lst]
	[(< (length lst) n)
      	  (append
	    out-lst
	    (list (wiki-heading 3 (string-append label " " (->string i))))
	    lst)]
	[else
	  (loop
	    (+ i 1) 
	    (drop lst n)
	    (append
	      out-lst
	      (list (wiki-heading 3 (string-append label " " (->string i))))
	      (take lst n)))]))))


(define (delete-adjacent-duplicates lst . opt)
  (let ((eq-pred (if (null? opt) equal? (car opt))))
    (fold-right
      (lambda (elem ret)
	(if (eq-pred elem (first ret))
	ret
	(cons elem ret)))
      (list (last lst))
      lst)))

(define (pages-containing-string wiki string exclusion-list)
  (let* (
    [possible-matches
      (delete-adjacent-duplicates
	(sort
	  (filter
	    (lambda (page) (not (member page exclusion-list)))
	    (search (string-append "\"" string "\"") 50000))
	  string<?)
	string=?)]
    [matches
      (filter
	(lambda (page)
	  (page-contains-pattern wiki page
            (make-pattern string)))
	possible-matches)])
    matches))

(define (wiki-page->exclusion-list wiki-content)
  (and wiki-content
    (map wiki-unlink
      (wiki-list->list
        (wiki-remove-list-preamble
          (string-split
            (wiki-clean wiki-content) "\n" #t))))))

(let* (
       [search-string
	  (with-input-from-string (second (argv)) read-string)]
       [output-page
          (if (> (length (argv)) 2)
	    (wiki-unlink (with-input-from-string (third (argv)) read-string))
            (string-append "User:" client-username "/searches/" search-string))]
       [exclusion-list-page
          (and (> (length (argv)) 3)
             (wiki-unlink
               (with-input-from-string (fourth (argv)) read-string)))]
       [exclusion-list (if exclusion-list-page (wiki-page->exclusion-list (wiki-page-contents wiki exclusion-list-page)) '())]
       [result (pages-containing-string wiki search-string exclusion-list)]
       [link-function
         (if (> (length result) 1000)
         wiki-link
         (lambda (x) (string-append "{{la|" x "}}")))])
  (write-wiki-page
    output-page
    (string-append
       (wiki-heading 2 (string-append "Pages unexpectedly containing the phrase \"" search-string "\""))
       (big-apply string-append
         (wiki-section-every 100
           (wiki-list (map link-function result)))))
    (string-append
      "List of pages unexpectedly containing the phrase \""
      search-string "\" "
      "compiled for " client-username " by " bot-username " using " (irnc-user-agent))))