scala-news-reader

FeedEntry.scala

(6323B)

 package com.joereader.lib.rss
 
 import com.sun.syndication._
 import feed.synd._
 import feed.module.mediarss._, types._
 
 import org.jsoup._
 import nodes._
 import select._
 import parser._
 import safety._
 
 import net.liftweb.util.PCDataXmlParser
 
 import scala.xml.NodeSeq
 import scala.collection.JavaConversions._
 import java.util.Date
 import java.text.SimpleDateFormat
 
 import com.joereader.lib._
 
 import net.liftweb.common._
 
 /* An RSS Feed Entry/Item. */
 case class FeedEntry(
                       guid: String,
                       title: String,
                       link: String,
                       date: Date,
                       author: FeedAuthor,
                       images: List[FeedImage],
                       content: NodeSeq) {
 
   private def df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")
   def dateFormatted = df.format(date)
   def nonAuthorImages = images.filterNot(i => author.imgUrl.exists(_ == i.src))
 }
 
 object FeedEntry extends Logger {
 
   /*
    * We cheat when possible and remove the following query params from image
    * sources.
    */
   val invalidImgParams = List(
       "w", "width", "h", "height", "crop", "size", "s")
   
   /* Image sources containing the following keywords are not allowed. */
   val invalidImgSources = List(
       "feeds.feedburner.com", "feedsportal.com", "subscribe")
 
   def build(entry: SyndEntry) = {
     val builtContent = content(entry)
     val builtMediaImages = mediaImages(entry, builtContent)
 
     FeedEntry(
       guid(entry),
       entry.getTitle,
       entry.getLink,
       entry.getPublishedDate,
       author(entry, builtMediaImages),
       builtMediaImages,
       builtContent)
   }
   
   def empty = FeedEntry("","","", new Date(), FeedAuthor(""), Nil, NodeSeq.Empty)
 
   def build(entries: List[SyndEntry]): List[FeedEntry] = entries.map(build)
 
   def build(entries: Option[List[SyndEntry]]): List[FeedEntry] =
     entries.map(build).getOrElse(Nil)
 
   /* Create FeedAuthor */
   def author(entry: SyndEntry, images: List[FeedImage]): FeedAuthor = {
     import gdata._
 
     // grab image src located in Author GData namespace (used by blogger.com)
     val gdataImage = entry.getAuthors.asInstanceOf[java.util.List[SyndPerson]].
       headOption.map {
       person =>
         val mod = person.getModule(GDataModule.URI).asInstanceOf[GDataModule]
         if (mod != null) mod.getThumbnail else ""
     }
 
     val gravatarImage: Option[String] =
       images.find(_.src.contains("gravatar.com")).map(_.src)
 
     val img = if(gravatarImage.isDefined) gravatarImage else gdataImage
 
     FeedAuthor(entry.getAuthor, img)
   }
 
   /* Find All images in entry. */
   def mediaImages(entry: SyndEntry, content: NodeSeq): List[FeedImage] = ({
     val mediaMod: MediaEntryModule = entry.getModule(MediaModule.URI).
       asInstanceOf[MediaEntryModule]
 
     if (mediaMod != null)
       (for (mediaContent <- mediaMod.getMediaContents) yield {
         val url = mediaContent.getReference.asInstanceOf[UrlReference].getUrl
         val src =
           if (mediaContent.getMedium == "image")
             URLFormatter(url.toString).
               urlWithoutQueryParams(invalidImgParams: _*)
           else
             ""
         FeedImage(src)
       }).toList
     else Nil
   } ::: {
     (content \\ "img").map {
       img =>
         val width = (img \ "@width").text
         val height = (img \ "@height").text
         val src = (img \ "@src").text
 
         try {
           if ((width == "" || width.toInt > 25) &&
             (height == "" || height.toInt > 25))
             FeedImage(src)
           else
             FeedImage("")
         } catch {
           case e: Throwable => 
             warn("Finding image: "+ e.getMessage)
             FeedImage("")
         }
 
     }.toList
 
   }).
     filterNot(_.src == "").
     filterNot(i =>
       invalidImgSources.exists(i.src.contains)
     ). // feedburner's bottom image links
     groupBy(_.src).
     map(src => FeedImage(src._1)).
     toList
 
   /* Grabs and formats the entry's content. */
   def content(entry: SyndEntry): NodeSeq = {
     def content = entry.getContents.asInstanceOf[java.util.List[SyndContent]].
       headOption.map(_.getValue).getOrElse(entry.getDescription.getValue)
 
     def whitelist = Whitelist.relaxed.
       addTags("iframe").
       addAttributes("iframe", "src", "frameborder", "allowfullscreen").
       addEnforcedAttribute("a", "rel", "nofollow").
       addEnforcedAttribute("a", "target", "_blank")
 
     def unsanitized = Jsoup.parse("<div>" + content + "</div>").body
     def imgUnsanitized = formatImages(unsanitized)
     def vidUnsanitized = formatVideos(imgUnsanitized)
     def semisanitized = Jsoup.clean(vidUnsanitized.toString, whitelist)
     def sanitized = Jsoup.parse(semisanitized).body.children.addClass("content")
     PCDataXmlParser(sanitized.toString) openOr NodeSeq.Empty
   }
 
   def formatImages(content: Element): Element = {
     val imgs: Elements = content.select("img")
 
     for (img <- imgs) {
       try {
         val link = URLFormatter(img.attr("src")).
           urlWithoutQueryParams(invalidImgParams: _*)
 
         img.attr("src", link)
 
         val w = "width"
         val h = "height"
         if (img.attr(w) != "" && img.attr(w).toInt > 25) img.attr(w, "")
         if (img.attr(h) != "" && img.attr(h).toInt > 25) img.attr(h, "")
       } catch {
         case e: Throwable =>
           warn("Formatting image: "+ e.getMessage)
       }
     }
     content
   }
 
   def formatVideos(content: Element): Element = {
 
     def emptyElem = new Element(Tag.valueOf("span"), "")
     val iframes: Elements = content.select("iframe")
 
     // only allow embedded youtube, vimeo videos
     val validSrc = 
       "http://www.youtube.com/embed/" :: 
       "http://player.vimeo.com/video/" :: Nil
 
     for (iframe <- iframes) {
       try {
         val src = iframe.attr("src")
         if (validSrc.exists(s => src.startsWith(s))) iframe.html("")
         else iframe.replaceWith(emptyElem)
       } catch {
         case e: Throwable =>
           warn("Formatting video: "+ e.getMessage)
       }
     }
     content
   }
 
   /**
    * GUID. If not provided, we create our own with 
    * title and published date.
    */
   def guid(entry: SyndEntry): String = {
     if (entry.getUri == "" || entry.getUri == null)
       entry.getTitle + entry.getPublishedDate
     else
       entry.getUri
   }
 }