scala-news-reader
rss/atom news reader in scala
git clone https://9o.is/git/scala-news-reader.git
commit fc3c63640903bf4f625f19c2bdc9bf3d1d317387 parent e2c3440a9da734d3e43d5672ccbe87bf961554c9 Author: Jul <jul@9o.is> Date: Wed, 7 Aug 2013 23:40:10 -0400 rss library now fully equipped for futures Diffstat:
13 files changed, 376 insertions(+), 245 deletions(-)
diff --git a/src/main/scala/com/joereader/lib/rss/Feed.scala b/src/main/scala/com/joereader/lib/rss/Feed.scala @@ -6,28 +6,34 @@ import scala.collection.SortedSet /* An RSS Feed. */ case class Feed(name: String, - description: String, - imageUrl: Option[String], - writers: List[FeedAuthor], - entries: List[FeedEntry], - links: List[String]) + description: String, + imageUrl: Option[String], + writers: List[FeedAuthor], + entries: List[FeedEntry], + links: List[String]) object Feed { - def build(feed: Option[SyndFeed], links: List[String]) = { + def build(feed: SyndFeed, links: List[String]) = { - val entries = FeedEntry.build(feed.map(_.getEntries. - asInstanceOf[java.util.List[SyndEntry]].toList)) + val entries = FeedEntry.build(feed.getEntries. + asInstanceOf[java.util.List[SyndEntry]].toList) + + def image = + if (feed.getImage != null) + Some(feed.getImage.getUrl) + else None Feed( - feed.map(_.getTitle).getOrElse(""), - feed.map(_.getDescription).getOrElse(""), - feed.map(f => if (f.getImage != null) f.getImage.getUrl else ""), + feed.getTitle, + feed.getDescription, + image, entries.map(_.author).distinct, entries, - links - ) + links) } + + def empty = Feed("", "", None, Nil, Nil, Nil) /** * Merges multiple feeds into one. If any feed contains one entry with same @@ -87,9 +93,7 @@ object Feed { _entries, _links) - } - else - Feed("", "", None, Nil, Nil, Nil) + } else Feed.empty } case class FeedImage(src: String) diff --git a/src/main/scala/com/joereader/lib/rss/FeedEntry.scala b/src/main/scala/com/joereader/lib/rss/FeedEntry.scala @@ -19,6 +19,8 @@ import java.text.SimpleDateFormat import com.joereader.lib._ +import net.liftweb.common._ + /* An RSS Feed Entry/Item. */ case class FeedEntry( id: String, @@ -33,13 +35,18 @@ case class FeedEntry( def nonAuthorImages = images.filterNot(i => author.imgUrl.exists(_ == i.src)) } -object FeedEntry { +object FeedEntry extends Logger { /* * We cheat when possible and remove the following query params from image * sources. */ - val invalidImgParams = List("w", "width", "h", "height", "crop", "size", "s") + val invalidImgParams = List( + "w", "width", "h", "height", "crop", "size", "s") + + /* Image sources containing the following keywords are not allowed. */ + val invalidImgSources = List( + "feeds.feedburner.com", "feedsportal.com", "subscribe") def build(entry: SyndEntry) = { val builtContent = content(entry) @@ -111,7 +118,9 @@ object FeedEntry { else FeedImage("") } catch { - case _: Throwable => FeedImage("") + case e: Throwable => + warn("Finding image: "+ e.getMessage) + FeedImage("") } }.toList @@ -119,9 +128,7 @@ object FeedEntry { }). filterNot(_.src == ""). filterNot(i => - i.src.contains("feeds.feedburner.com") || - i.src.contains("feedsportal.com") || - i.src.contains("subscribe") + invalidImgSources.exists(i.src.contains) ). // feedburner's bottom image links groupBy(_.src). map(src => FeedImage(src._1)). @@ -129,20 +136,20 @@ object FeedEntry { /* Grabs and formats the entry's content. */ def content(entry: SyndEntry): NodeSeq = { - val content = entry.getContents.asInstanceOf[java.util.List[SyndContent]]. + def content = entry.getContents.asInstanceOf[java.util.List[SyndContent]]. headOption.map(_.getValue).getOrElse(entry.getDescription.getValue) - val whitelist = Whitelist.relaxed. + def whitelist = Whitelist.relaxed. addTags("iframe"). addAttributes("iframe", "src", "frameborder", "allowfullscreen"). addEnforcedAttribute("a", "rel", "nofollow"). addEnforcedAttribute("a", "target", "_blank") - val unsanitized = Jsoup.parse("<div>" + content + "</div>").body - val imgUnsanitized = formatImages(unsanitized) - val vidUnsanitized = formatVideos(imgUnsanitized) - val semisanitized = Jsoup.clean(vidUnsanitized.toString, whitelist) - val sanitized = Jsoup.parse(semisanitized).body.children.addClass("content") + def unsanitized = Jsoup.parse("<div>" + content + "</div>").body + def imgUnsanitized = formatImages(unsanitized) + def vidUnsanitized = formatVideos(imgUnsanitized) + def semisanitized = Jsoup.clean(vidUnsanitized.toString, whitelist) + def sanitized = Jsoup.parse(semisanitized).body.children.addClass("content") PCDataXmlParser(sanitized.toString) openOr NodeSeq.Empty } @@ -161,7 +168,8 @@ object FeedEntry { if (img.attr(w) != "" && img.attr(w).toInt > 25) img.attr(w, "") if (img.attr(h) != "" && img.attr(h).toInt > 25) img.attr(h, "") } catch { - case _: Throwable => + case e: Throwable => + warn("Formatting image: "+ e.getMessage) } } content @@ -173,7 +181,9 @@ object FeedEntry { val iframes: Elements = content.select("iframe") // only allow embedded youtube, vimeo videos - val validSrc = "http://www.youtube.com/embed/" :: "http://player.vimeo.com/video/" :: Nil + val validSrc = + "http://www.youtube.com/embed/" :: + "http://player.vimeo.com/video/" :: Nil for (iframe <- iframes) { try { @@ -181,14 +191,16 @@ object FeedEntry { if (validSrc.exists(s => src.startsWith(s))) iframe.html("") else iframe.replaceWith(emptyElem) } catch { - case _: Throwable => + case e: Throwable => + warn("Formatting video: "+ e.getMessage) } } content } /** - * GUID. If not provided, we create our own with title and published date. + * GUID. If not provided, we create our own with + * title and published date. */ def guid(entry: SyndEntry): String = { if (entry.getUri == "" || entry.getUri == null) diff --git a/src/main/scala/com/joereader/lib/rss/MetaVerification.scala b/src/main/scala/com/joereader/lib/rss/MetaVerification.scala @@ -1,13 +1,26 @@ package com.joereader.lib.rss import dispatch._ +import net.liftweb.common._ /* Verifies if a webpage contains a specific meta tag. */ -case class MetaVerification(metaName: String, metaContent: String, link: String) { +case class MetaVerification( + metaName: String, + metaContent: String, + link: String) extends Logger { def verified: Boolean = - requestLink(link)().exists(response => - headElements(parseHtml(response)) exists - (e => e.attr("name").toLowerCase.trim == metaName.toLowerCase.trim && - e.attr("content").toLowerCase.trim == metaContent.toLowerCase.trim)) + requestLink(link)() match { + case Left(e) => + warn("Can't connect to verify meta tag: "+e.getMessage) + false + case Right(response) => + headElements(parseHtml(response)) exists + (e => + e.attr("name").toLowerCase.trim == + metaName.toLowerCase.trim && + e.attr("content").toLowerCase.trim == + metaContent.toLowerCase.trim) + + } } \ No newline at end of file diff --git a/src/main/scala/com/joereader/lib/rss/package.scala b/src/main/scala/com/joereader/lib/rss/package.scala @@ -10,31 +10,38 @@ import java.io._ import scala.Some import scala.collection.JavaConversions._ +import net.liftweb._ +import common._ +import util.Helpers._ + case class RSSHtmlResponse(loc: String, content: String, redirectTo: String) /** * */ -package object rss { +package object rss extends Logger { implicit class RssImplicitString(str: String) { /** * Grabs all feed entries given a url to a rss feed. */ - def entries: List[FeedEntry] = { - val response = requestLink(str)() - Feed.build(extractFeed(response), List(str)).entries + def entries: Future[Either[String, List[FeedEntry]]] = { + val response = catchThrowable(requestLink(str)) + + for(res <- extractFeed(response).right) + yield Feed.build(res, List(str)).entries } /** * Returns html given a url to a web page */ - def response: RSSHtmlResponse = { + def response: Future[Either[String, RSSHtmlResponse]] = { val req = RSSHtmlResponse(str, "", str) - val optReq: Option[RSSHtmlResponse] = Some(req) - val res = requestLink(optReq) - res.getOrElse(req) + Future { + requestLink(Some(req)). + toRight("Failed to get html response") + } } /** @@ -51,14 +58,22 @@ package object rss { * Given a url link to rss feed, it returns a syndicated feed. * @return a syndicated feed */ - def feed: Feed = { - val feeds = strs.map { - link => - val response = requestLink(link)() - Feed.build(extractFeed(response), List(link)) - }.toList - Feed.merge(feeds) - } + def feed: Feed = + Feed.merge(strs. + map(l => (catchThrowable(requestLink(l)), l)). + map { + r => + val (response, link) = r + for (syndFeed <- extractFeed(response).right) + yield Feed.build(syndFeed, List(link)) + }.map( + _() match { + case Left(msg) => + warn(msg) + Feed.empty + case Right(feed) => + feed + })) } /** @@ -66,12 +81,12 @@ package object rss { * @param link the web link to the page. * @return either a failure with a message or successful content as a string */ - def requestLink(link: String): Future[Option[String]] = { + def requestLink(link: String): Future[Either[Throwable, String]] = { val request = url(URLFormatter(link).toString) Http.configure(_. setFollowRedirects(true). - setCompressionEnabled(true))(request OK as.String).option + setCompressionEnabled(true))(request OK as.String).either } /** @@ -100,6 +115,12 @@ package object rss { } } + private def catchThrowable(res: Future[Either[Throwable, String]]): Future[Either[String, String]] = + for (e <- res.left) yield { + error("Cannot connect to request RSS: " + e.getMessage) + e.getMessage + } + /** * Finds link(s) to RSS feeds in HTML elements. * @param html html in plain text. @@ -140,17 +161,19 @@ package object rss { * @param xml xml as string. * @return a syndicated feed if xml if correctly parsed. */ - def extractFeed(xml: String): Option[SyndFeed] = try { - val bytes = new ByteArrayInputStream(xml.getBytes("UTF-8")) - val reader = new XmlReader(bytes) - val feed = new SyndFeedInput().build(reader) - Some(feed) - } catch { - case _: Throwable => None - } - - def extractFeed(link: Option[String]): Option[SyndFeed] = - link.map(extractFeed).flatten + def extractFeed(xml: String): Either[String, SyndFeed] = + tryo { + val bytes = new ByteArrayInputStream(xml.getBytes("UTF-8")) + val reader = new XmlReader(bytes) + new SyndFeedInput().build(reader) + } toRight "Unable to extract feed." + + def extractFeed(content: Future[Either[String, String]]): Future[Either[String, SyndFeed]] = + for (either <- content) + yield for { + feedStr <- either.right + out <- extractFeed(feedStr).right + } yield out def parseHtml(html: String): Document = Jsoup.parse(html) diff --git a/src/main/scala/com/joereader/snippet/ArticleSnip.scala b/src/main/scala/com/joereader/snippet/ArticleSnip.scala @@ -13,6 +13,9 @@ import config._ import lib.rss._ import model._ +import dispatch._ +import Defaults._ + import xml._ import java.util.Date @@ -20,24 +23,35 @@ import java.util.Date * Mix this in to list articles. */ trait ArticleSnip { - - def sort(entries: List[Article]) = - entries.sortWith((x,y) => x.date.getTime > y.date.getTime) - - def articles(articles: List[Article]): CssSel = - if(articles.isEmpty) - ".reader-nav [class+]" #> "hide" & - ".article *" #> - <p style="padding: 200px 0; text-align: center"> - No articles to display here - </p> + + def sort(entries: List[Article]) = + entries.sortWith((x, y) => x.date.getTime > y.date.getTime) + + def parse(articles: Future[Either[String, CssSel]]) = + articles() match { + case Left(msg) => + warn(msg) + noArticles + case Right(a) => a + } + + def noArticles: CssSel = + ".reader-nav [class+]" #> "hide" & + ".article *" #> + <p style="padding: 200px 0; text-align: center"> + No articles to display here + </p> + + def articles(articles: Future[Either[String, List[Article]]]) = + for (articles <- articles.right) + yield if (articles.isEmpty) noArticles else ".article *" #> sort(articles).map { article => def bwu = article.bwu def entry = article.entry - def shareId = article.id.hashCode+"-share" - def saveId = article.id.hashCode+"-save" + def shareId = article.id.hashCode + "-share" + def saveId = article.id.hashCode + "-save" def color = "color:" + bwu.color def saved: Boolean = User.currentUser.exists( @@ -46,26 +60,26 @@ trait ArticleSnip { def shared: Boolean = User.currentUser.exists( _.shared.get.exists(_ == article.id)) - def saveLink: NodeSeq = + def saveLink: NodeSeq = a(() => save, Text("Save"), "id" -> saveId, "style" -> color) - - def unSaveLink: NodeSeq = + + def unSaveLink: NodeSeq = a(() => unSave, Text("Undo Save"), "id" -> saveId, "style" -> color) - def shareLink: NodeSeq = + def shareLink: NodeSeq = a(() => share, Text("Share"), "id" -> shareId, "style" -> color) - - def unShareLink: NodeSeq = + + def unShareLink: NodeSeq = a(() => unShare, Text("Undo Share"), "id" -> shareId, "style" -> color) def save: JsCmd = { - for (u <- User.currentUser) + for (u <- User.currentUser) u.saveArticle(article).update Replace(saveId, unSaveLink) } def unSave: JsCmd = { - for (u <- User.currentUser) + for (u <- User.currentUser) u.unSaveArticle(article).update Replace(saveId, saveLink) } @@ -79,15 +93,15 @@ trait ArticleSnip { } def unShare: JsCmd = { - for (u <- User.currentUser) + for (u <- User.currentUser) u.unShareArticle(article.asInstanceOf[SharedArticle]).update Replace(shareId, shareLink) } - ".article-save" #> (if(User.isLoggedIn) {if (saved) unSaveLink else saveLink} else NodeSeq.Empty) & - ".article-share" #> (if(User.isLoggedIn) {if (shared) unShareLink else shareLink} else NodeSeq.Empty) & + ".article-save" #> (if (User.isLoggedIn) { if (saved) unSaveLink else saveLink } else NodeSeq.Empty) & + ".article-share" #> (if (User.isLoggedIn) { if (shared) unShareLink else shareLink } else NodeSeq.Empty) & ".article-inner [style]" #> ("border-right:3px solid " + bwu.color) & - ".left-sub-header-inner [class+]" #> (if(article.sharedBy.isDefined) "" else "hide") & + ".left-sub-header-inner [class+]" #> (if (article.sharedBy.isDefined) "" else "hide") & ".article-sharedby *" #> article.sharedBy.map(_.name.get) & ".article-sharedby [href]" #> article.sharedBy.map(Site.userProfileLoc.calcHref) & ".article-sharedby-img [src]" #> BlogWriterUser(article.sharedBy).image & @@ -117,41 +131,41 @@ trait ArticleSnip { trait ArticleTrait { def separator = "~" - def createId(blog: Blog, entry: FeedEntry) = + def createId(blog: Blog, entry: FeedEntry) = blog.id.get + separator + entry.id } object Article extends ArticleTrait object SharedArticle extends ArticleTrait { - def createId(blog: Blog, entry: FeedEntry, date: Date) = + def createId(blog: Blog, entry: FeedEntry, date: Date) = super.createId(blog, entry) + separator + date.getTime } class Article(_bwu: BlogWriterUser, _entry: FeedEntry) { - - def id: String = + + def id: String = bwu.blog.map(Article.createId(_, entry)). - getOrElse("") - + getOrElse("") + def date: Date = entry.date def entry = _entry def bwu = _bwu - + val sharedBy: Option[User] = None } class SharedArticle( - bwu: BlogWriterUser, - entry: FeedEntry, - _sharedBy: User, - sharedDate: Date = new Date) extends Article(bwu, entry) { - + bwu: BlogWriterUser, + entry: FeedEntry, + _sharedBy: User, + sharedDate: Date = new Date) extends Article(bwu, entry) { + override def date = sharedDate - override def id = + override def id = bwu.blog.map(SharedArticle.createId(_, entry, date)). - getOrElse("") - + getOrElse("") + override val sharedBy = Some(_sharedBy) } diff --git a/src/main/scala/com/joereader/snippet/BlogSnipView.scala b/src/main/scala/com/joereader/snippet/BlogSnipView.scala @@ -11,6 +11,7 @@ import snippet.SnipHelpers._ import model._ import scala.xml._ +import dispatch._, Defaults._ /** * Snippets for the public to view Blog model information. @@ -32,7 +33,7 @@ trait BlogSnipView extends BlogSnip with ArticleSnip with BackgroundSnip { def url(html: NodeSeq) = serve(html) { blog => "a *" #> blog.urlHtml.get & - "a [href]" #> Site.blogProfileLoc.calcHref(blog) + "a [href]" #> Site.blogProfileLoc.calcHref(blog) }(test, NodeSeq.Empty) def description = serve { @@ -54,16 +55,19 @@ trait BlogSnipView extends BlogSnip with ArticleSnip with BackgroundSnip { def articles(html: NodeSeq): NodeSeq = serve(html) { blog => - val entries = blog.urlRss.is.head.entries.filter( - e => blog.writerExists(e.author.name)).map { - entry => - val blogWriter: Box[BlogWriter] = blog.writer(entry.author.name) - val bwu = BlogWriterUser( - blogWriter.flatMap(_.user.obj), Some(blog), blogWriter) - new Article(bwu, entry) - } + def url = blog.urlRss.get.head + val entries = + for (entries <- url.entries.right) + yield entries.filter( + e => blog.writerExists(e.author.name)).map { + entry => + val blogWriter: Box[BlogWriter] = blog.writer(entry.author.name) + val bwu = BlogWriterUser( + blogWriter.flatMap(_.user.obj), Some(blog), blogWriter) + new Article(bwu, entry) + } - articles(entries) + parse(articles(entries)) }(test, NodeSeq.Empty) } diff --git a/src/main/scala/com/joereader/snippet/BlogSnips.scala b/src/main/scala/com/joereader/snippet/BlogSnips.scala @@ -13,7 +13,7 @@ import com.joereader.model._ import config._ -trait BlogSnip extends SnippetHelper with Loggable { +trait BlogSnip extends SnippetHelper with Logger { protected def blog: Box[Blog] @@ -35,7 +35,7 @@ trait BlogSnip extends SnippetHelper with Loggable { }): NodeSeq } -trait BlogWriterSnip extends SnippetHelper with Loggable { +trait BlogWriterSnip extends SnippetHelper with Logger { protected def blog: Box[Blog] diff --git a/src/main/scala/com/joereader/snippet/BlogWriterSnipView.scala b/src/main/scala/com/joereader/snippet/BlogWriterSnipView.scala @@ -3,6 +3,9 @@ package com.joereader.snippet import net.liftweb._ import util._, Helpers._ +import dispatch._ +import Defaults._ + import com.joereader._ import config._ import lib.rss._ @@ -18,11 +21,11 @@ trait BlogWriterSnipView extends BlogWriterUserSnip with ArticleSnip with Backgr def articles(html: NodeSeq): NodeSeq = serve(html) { (blog, blogWriter) => - val entries = blog.urlRss.get.head.entries. + val entries = blog.urlRss.get.head.entries.right.map(_. filter(_.author.name == blogWriter.name.get). - map(new Article(BlogWriterUser(None, Some(blog), Some(blogWriter)), _)) + map(new Article(BlogWriterUser(None, Some(blog), Some(blogWriter)), _))) - articles(entries) + parse(articles(entries)) }(test = true, NodeSeq.Empty) diff --git a/src/main/scala/com/joereader/snippet/MyBlog.scala b/src/main/scala/com/joereader/snippet/MyBlog.scala @@ -5,6 +5,9 @@ import model._ import config._ import lib.rss._ +import dispatch._ +import Defaults._ + import scala.xml._ import net.liftweb._ @@ -26,7 +29,7 @@ class MyBlog extends BlogSnip with ArticleSnip { blog => val entry = - blog.urlRss.is.head.entries. + blog.urlRss.is.head.entries.right.map(_. find(e => blog.writerExists(e.author.name)).map { entry => @@ -34,9 +37,9 @@ class MyBlog extends BlogSnip with ArticleSnip { val bwu = BlogWriterUser( blogWriter.flatMap(_.user.obj), Some(blog), blogWriter) new Article(bwu, entry) - } + }.toList) - articles(entry.toList) + parse(articles(entry)) }(test = true, NodeSeq.Empty) } diff --git a/src/main/scala/com/joereader/snippet/ReaderSnip.scala b/src/main/scala/com/joereader/snippet/ReaderSnip.scala @@ -9,6 +9,9 @@ import com.joereader._ import model._ import lib.rss._ +import dispatch._ +import Defaults._ + import scala.xml._ import java.util.Date @@ -21,16 +24,21 @@ class ReaderSnip extends UserSnip with ArticleSnip { def articles(html: NodeSeq): NodeSeq = serve(html) { user => - val articlesFollowing: List[Article] = + val articlesFollowing: List[Future[Either[String, List[Article]]]] = user.following.allUsers.flatMap(bwu => for { blog <- bwu.blog blogWriter <- bwu.blogWriter - } yield blog.urlRss.get.head.entries. - filter(_.author.name == blogWriter.name.get.toString). - map(new Article(bwu, _))).flatten - - val articlesShared: List[Article] = { + } yield { + val url = blog.urlRss.get.head + url.entries.right.map( entries => + entries.filter(_.author.name == blogWriter.name.get.toString). + map(new Article(bwu, _))) + } + ) + + + val articlesShared: List[Future[Either[String, List[Article]]]] = { user.following.usersSharedArticles.flatMap { user => val blogIds: List[String] = @@ -40,17 +48,20 @@ class ReaderSnip extends UserSnip with ArticleSnip { val blogs: List[Blog] = blogIds.flatMap(Blog.findByStringId) - val allEntries: List[(Blog, List[FeedEntry])] = - blogs.map(b => (b, b.urlRss.get.head.entries)).toList + val allEntries: List[(Blog, Future[Either[String, List[FeedEntry]]])] = + blogs.map(b => (b, b.urlRss.get.head.entries)) - val sharedEntries: List[(Blog, List[FeedEntry])] = + val sharedEntries: List[(Blog, Future[Either[String, List[FeedEntry]]])] = allEntries.map(f => - (f._1, f._2.filter { e => + (f._1, f._2.right.map(_.filter { e => user.shared.get.exists(_.startsWith(Article.createId(f._1, e))) - })).filterNot(_._2.isEmpty) + }))).filterNot(_._2.right.map(_.isEmpty)() match { + case Left(_) => false + case Right(b) => b + }) - sharedEntries.flatMap(f => - f._2.map { entry => + sharedEntries.map(f => + f._2.right.map(_.map { entry => val id = user.shared.get.find(_.startsWith(Article.createId(f._1, entry))) val time = id.map(_.split(SharedArticle.separator).last) val date: Option[Date] = tryo(time.map(t => new Date(t.toLong))).openOr(None) @@ -59,43 +70,61 @@ class ReaderSnip extends UserSnip with ArticleSnip { val blogWriter = f._1.writer(entry.author.name) val u = blogWriter.flatMap(_.user.obj) new SharedArticle(BlogWriterUser(u, blog, blogWriter), entry, user, date.getOrElse(entry.date)) - }) + })) } } + + def mergeEntries(entries: List[Future[Either[String, List[Article]]]]): List[Article] = + entries.flatMap(_() match { + case Left(msg) => + warn(msg); Nil + case Right(entries) => entries + }) - articles(articlesFollowing ::: articlesShared) + parse(articles(Future(Right(mergeEntries(articlesFollowing) ::: mergeEntries(articlesShared))))) }(TestUser.isLoggedIn, NodeSeq.Empty) def savedArticles(html: NodeSeq): NodeSeq = serve(html) { user => - def entries: List[Article] = { + def entries: List[Future[Either[String, List[Article]]]] = { val blogIds: List[String] = user.saved.get.flatMap(str => - str.split(Article.separator).headOption) + str.split(Article.separator).headOption).distinct val blogs: List[Blog] = blogIds.flatMap(Blog.findByStringId) - val allEntries: List[(Blog, List[FeedEntry])] = - blogs.map(b => (b, b.urlRss.get.head.entries)).toList + val allEntries: List[(Blog, Future[Either[String, List[FeedEntry]]])] = + blogs.map(b => (b, b.urlRss.get.head.entries)) - val savedEntries: List[(Blog, List[FeedEntry])] = + val savedEntries: List[(Blog, Future[Either[String, List[FeedEntry]]])] = allEntries.map(f => - (f._1, f._2.filter(e => user.saved.get. - exists(_ == Article.createId(f._1, e))))).filterNot(_._2.isEmpty) + (f._1, f._2.right.map(_.filter(e => user.saved.get. + exists(_ == Article.createId(f._1, e)))))). + filterNot(_._2.right.map(_.isEmpty)() match { + case Left(_) => false + case Right(b) => b + }) - savedEntries.flatMap(f => - f._2.map { entry => + savedEntries.map(f => + f._2.right.map(_.map { entry => val blog = Some(f._1) val blogWriter = f._1.writer(entry.author.name) val user = blogWriter.flatMap(_.user.obj) new Article(BlogWriterUser(user, blog, blogWriter), entry) - }) + })) } - articles(entries) + def mergeEntries(entries: List[Future[Either[String, List[Article]]]]): List[Article] = + entries.flatMap(_() match { + case Left(msg) => + warn(msg); Nil + case Right(entries) => entries + }) + + parse(articles(Future(Right(mergeEntries(entries))))) }(TestUser.isLoggedIn, NodeSeq.Empty) diff --git a/src/main/scala/com/joereader/snippet/UserSnips.scala b/src/main/scala/com/joereader/snippet/UserSnips.scala @@ -15,7 +15,7 @@ import model._ import config._ -trait UserSnip extends SnippetHelper with Loggable { +trait UserSnip extends SnippetHelper with Logger { protected def user: Box[User] diff --git a/src/main/scala/com/joereader/snippet/UserWriterSnipView.scala b/src/main/scala/com/joereader/snippet/UserWriterSnipView.scala @@ -4,6 +4,8 @@ import net.liftweb.util.Helpers._ import scala.xml._ import java.util.Date +import dispatch._, Defaults._ + import com.joereader._ import config._ import lib.rss._ @@ -27,7 +29,7 @@ trait UserWriterSnipView extends UserSnip with ArticleSnip with BackgroundSnip { user => "* [id]" #> imgBgId & - "* [src]" #> imageBgUrl(user) + "* [src]" #> imageBgUrl(user) }(test, NodeSeq.Empty) def followingList(html: NodeSeq) = serve(html) { @@ -35,8 +37,8 @@ trait UserWriterSnipView extends UserSnip with ArticleSnip with BackgroundSnip { val following = user.following.randomUsers(6) "*" #> following.map(bwu => - <a href={bwu.link}> - <img src={bwu.image}/> + <a href={ bwu.link }> + <img src={ bwu.image }/> </a>) }(test, NodeSeq.Empty) @@ -45,8 +47,8 @@ trait UserWriterSnipView extends UserSnip with ArticleSnip with BackgroundSnip { val following: Int = user.following.get.size if (following > 0) "*" #> - <a href={Site.userFollowingLoc.calcHref(user)}> - Following {following} + <a href={ Site.userFollowingLoc.calcHref(user) }> + Following{ following } </a> else "*" #> NodeSeq.Empty @@ -55,46 +57,61 @@ trait UserWriterSnipView extends UserSnip with ArticleSnip with BackgroundSnip { def articles(html: NodeSeq): NodeSeq = serve(html) { user => - val entries: List[Article] = - (for { + val entries: List[Future[Either[String, List[Article]]]] = + for { blogs <- user.blogs.get blog <- Blog.find(blogs) blogWriter <- blog.writer(user) - } yield blog.urlRss.get.head.entries. - filter(_.author.name == blogWriter.name.get). - map(new Article(BlogWriterUser(Some(user), Some(blog), Some(blogWriter)), _))). - flatten - - val sharedEntries: List[SharedArticle] = { + } yield { + val url = blog.urlRss.get.head + url.entries.right.map(entries => + entries.filter(_.author.name == blogWriter.name.get). + map(new Article(BlogWriterUser( + Some(user), Some(blog), Some(blogWriter)), _))) + } + + val sharedEntries: List[Future[Either[String, List[SharedArticle]]]] = { val blogIds: List[String] = - user.shared.get.flatMap(str => - str.split(Article.separator).headOption).distinct - - val blogs: List[Blog] = - blogIds.flatMap(Blog.findByStringId) - - val allEntries: List[(Blog, List[FeedEntry])] = - blogs.map(b => (b, b.urlRss.get.head.entries)).toList - - val sharedEntries: List[(Blog, List[FeedEntry])] = - allEntries.map(f => - (f._1, f._2.filter { e => - user.shared.get.exists(_.startsWith(Article.createId(f._1, e))) - })).filterNot(_._2.isEmpty) - - sharedEntries.flatMap(f => - f._2.map { entry => - val id = user.shared.get.find(_.startsWith(Article.createId(f._1, entry))) - val time = id.map(_.split(SharedArticle.separator).last) - val date: Option[Date] = tryo(time.map(t => new Date(t.toLong))).openOr(None) - - val blog = Some(f._1) - val blogWriter = f._1.writer(entry.author.name) - val u = blogWriter.flatMap(_.user.obj) - new SharedArticle(BlogWriterUser(u, blog, blogWriter), entry, user, date.getOrElse(entry.date)) - }) + user.shared.get.flatMap(str => + str.split(Article.separator).headOption).distinct + + val blogs: List[Blog] = + blogIds.flatMap(Blog.findByStringId) + + val allEntries: List[(Blog, Future[Either[String, List[FeedEntry]]])] = + blogs.map(b => (b, b.urlRss.get.head.entries)) + + val sharedEntries: List[(Blog, Future[Either[String, List[FeedEntry]]])] = + allEntries.map(f => + (f._1, f._2.right.map { entries => + entries.filter(e => + user.shared.get.exists(_.startsWith(Article.createId(f._1, e)))) + })).filterNot(_._2.right.map(_.isEmpty)() match { + case Left(_) => false + case Right(b) => b + }) + + sharedEntries.map(f => + f._2.right.map(_.map { entry => + val id = user.shared.get.find(_.startsWith(Article.createId(f._1, entry))) + val time = id.map(_.split(SharedArticle.separator).last) + val date: Option[Date] = tryo(time.map(t => new Date(t.toLong))).openOr(None) + + val blog = Some(f._1) + val blogWriter = f._1.writer(entry.author.name) + val u = blogWriter.flatMap(_.user.obj) + new SharedArticle(BlogWriterUser(u, blog, blogWriter), entry, user, date.getOrElse(entry.date)) + })) } - articles(entries ::: sharedEntries) + + def mergeEntries(entries: List[Future[Either[String, List[Article]]]]): List[Article] = + entries.flatMap(_() match { + case Left(msg) => + warn(msg); Nil + case Right(entries) => entries + }) + + parse(articles(Future(Right(mergeEntries(entries) ::: mergeEntries(sharedEntries))))) }(test, NodeSeq.Empty) diff --git a/src/main/scala/com/joereader/snippet/Verification.scala b/src/main/scala/com/joereader/snippet/Verification.scala @@ -17,6 +17,7 @@ import model._ import scala.xml._ import dispatch._ +import Defaults._ /* * A session var had to be included so we can keep state between Wordpress @@ -36,7 +37,7 @@ case class VerificationSettings() { */ class Verification extends VerificationDesigns { - var s = VerificationSettings() + var set = VerificationSettings() val writerName = ValueCell("") @@ -45,7 +46,7 @@ class Verification extends VerificationDesigns { val metaName = "readmeans" val metaContent = (writerName lift owner)( - (w, o) => s.user.id.is + ":" + w + { + (w, o) => set.user.id.is + ":" + w + { if (o) ":owner" else "" }) @@ -61,32 +62,32 @@ class Verification extends VerificationDesigns { me <- WordpressClient.me().right site <- WordpressClient.site(me)().right } { - if (URLFormatter.same(s.urlHtml, site.URL.toString) && - s.blog.writersNames.exists(_ == me.username)) { - writerName.set(me.username) - s.verified = true - } + if (URLFormatter.same(set.urlHtml, site.URL.toString) && + set.blog.writersNames.exists(_ == me.username)) { + writerName.set(me.username) + set.verified = true + } } - s.verified = - if (s.verified) true - else MetaVerification(metaName, metaContent.get, s.urlHtml).verified + set.verified = + if (set.verified) true + else MetaVerification(metaName, metaContent.get, set.urlHtml).verified - if (s.verified) verifiedProcess() + if (set.verified) verifiedProcess() else S.error("Verification was unsuccessful.") } def verifiedProcess(): JsCmd = { // save blog stuff - val bw = BlogWriter.createRecord.name(writerName.is).user(s.user.id.is) + val bw = BlogWriter.createRecord.name(writerName.is).user(set.user.id.is) - if (owner) s.blog.owner(s.user.id.is) - s.blog.addWriterSafely(bw) + if (owner) set.blog.owner(set.user.id.is) + set.blog.addWriterSafely(bw) - if (s.blog.blogname.is.isEmpty) - s.blog.blogname(StringHelpers.randomString(15).toLowerCase) + if (set.blog.blogname.is.isEmpty) + set.blog.blogname(StringHelpers.randomString(15).toLowerCase) - s.blog.save + set.blog.save WordpressClient.resetToken // reset CurrentVerification(VerificationSettings()) @@ -97,34 +98,42 @@ class Verification extends VerificationDesigns { def verifyBlog(outer: IdMemoizeTransform) = { // if idmemoize updated, get urlhtml and find feed - if (s.urlHtml.nonEmpty) { - val existing = Blog.findByUrl(s.urlHtml) - val res = s.urlHtml.response - val links = res.content.rssLinks - val feed: Feed = links.feed - val writers = feed.writers.map { - w => BlogWriter.createRecord.name(w.name).img(w.imgUrl) - } + if (set.urlHtml.nonEmpty) { + val existing = Blog.findByUrl(set.urlHtml) + val res = set.urlHtml.response + + val links = res.right.map(_.content.rssLinks) + val feed = links.right.map(_.feed) + val writers = feed.right.map(_.writers.map( + w => + BlogWriter.createRecord. + name(w.name).img(w.imgUrl) + )) if (existing.isEmpty) { - s.blog = Blog.createRecord + for { + res <- res.right + feed <- feed.right + writers <- writers.right + } set.blog = Blog.createRecord .urlHtml(res.loc) .name(feed.name) .description(feed.description) .writers(writers) .urlRss(feed.links) } else { - existing.map(s.blog = _) - writers.map(writer => s.blog.addWriterSafely(writer)) + existing.map(set.blog = _) + writers.right.map(_.map( + writer => set.blog.addWriterSafely(writer))) } - s.searched = true - s.urlHtml = s.blog.urlHtml.get // nicely formatted - CurrentVerification(s) + set.searched = true + set.urlHtml = set.blog.urlHtml.get // nicely formatted + CurrentVerification(set) } else - s = CurrentVerification.is + set = CurrentVerification.is - "#blog-url" #> text(s.urlHtml, s.urlHtml = _) & + "#blog-url" #> text(set.urlHtml, set.urlHtml = _) & "#search-blog" #> ajaxSubmit("Search Blog", () => ajaxInvoke(outer.setHtml)) & "#writer-list *" #> listWriters & "#blog-owner-q-yes" #> radios(0) & @@ -140,10 +149,10 @@ class Verification extends VerificationDesigns { } def writersRadios = ajaxRadio[String]( - s.blog.unregisteredWritersNames, - Full(s.blog.unregisteredWritersNames headOr ""), { + set.blog.unregisteredWritersNames, + Full(set.blog.unregisteredWritersNames headOr ""), { s => onWritersRadiosChange(s) & onWritersRadiosChangeSignUp(s) - }).unregisteredWritersChoicesToPictureForm(s.blog) + }).unregisteredWritersChoicesToPictureForm(set.blog) val radios = ajaxRadio[Boolean]( Seq(true, false), @@ -154,34 +163,34 @@ class Verification extends VerificationDesigns { }) def registeredUsers = - s.blog.registeredWriters. - registeredWritersChoicesToPictureForm(s.blog) + set.blog.registeredWriters. + registeredWritersChoicesToPictureForm(set.blog) def completedMsg = - if (s.blog.unregisteredWriters.isEmpty && s.searched) + if (set.blog.unregisteredWriters.isEmpty && set.searched) Text("Looks like all writers have joined") else NodeSeq.Empty def hideVerification: String = { - s.blog.unregisteredWritersNames.nonEmpty ? "" | "hide" + set.blog.unregisteredWritersNames.nonEmpty ? "" | "hide" } def hideHelp: String = hideVerification def hideOwnerQuestion: String = { - val size = s.blog.unregisteredWritersNames.size + val size = set.blog.unregisteredWritersNames.size owner.set((size == 1) ? true | false) (size < 2) ? "hide" | "" } // returns the radio group of writers def listWriters(): NodeSeq = - if (s.blog.writersNames.isEmpty && s.searched) + if (set.blog.writersNames.isEmpty && set.searched) Text("Writers not found. Double check your url.") else { - writerName.set(s.blog.unregisteredWritersNames headOr "") + writerName.set(set.blog.unregisteredWritersNames headOr "") if (!EmailVar.is.isEmpty) { - s.user.name(writerName.is) + set.user.name(writerName.is) } writersRadios ++ registeredUsers ++ completedMsg } @@ -195,31 +204,31 @@ class Verification extends VerificationDesigns { if (!EmailVar.is.isEmpty) { BetaUser.find(EmailVar.is).map(_.delete_!) - s.user - .addBlog(s.blog) + set.user + .addBlog(set.blog) .email(EmailVar.is) .password(Helpers.randomString(20)) .username(StringHelpers.randomString(15).toLowerCase) - s.user.password.hashIt - s.user.save - User.logUserIn(s.user, isAuthed = true) + set.user.password.hashIt + set.user.save + User.logUserIn(set.user, isAuthed = true) RedirectTo(Site.signUp3.url, () => { - BlogIdVar(s.blog.id.is.toString) - VerifiedVar(s.verified) + BlogIdVar(set.blog.id.is.toString) + VerifiedVar(set.verified) }) } else Noop def onWritersRadiosChangeSignUp(str: String): JsCmd = if (!EmailVar.is.isEmpty) { writerName.set(str) - s.user.name(str) + set.user.name(str) Noop } else Noop protected def onVerified(): JsCmd = if (EmailVar.is.isEmpty) { - s.user.addBlog(s.blog).update + set.user.addBlog(set.blog).update if (owner) S.redirectTo(Site.editBlogs.url) else S.notice("Congratulations! You have verified your blog!")