From 98f9b7359476349ceeb11b864ee64a99a97963e3 Mon Sep 17 00:00:00 2001 From: Diana Belokon <67607236+dbelokon@users.noreply.github.com> Date: Fri, 10 Dec 2021 09:52:23 -0500 Subject: [PATCH] Implement videos as a type of posts --- src/api/posts/src/data/post.js | 5 ++++- src/api/posts/src/storage.js | 4 +++- src/backend/data/post.js | 24 +++++++++++++++++++++++- src/backend/feed/processor.js | 18 ++++++++++++++---- src/backend/index.js | 9 ++++++++- src/backend/utils/storage.js | 4 +++- 6 files changed, 55 insertions(+), 9 deletions(-) diff --git a/src/api/posts/src/data/post.js b/src/api/posts/src/data/post.js index aabeaecb98..cd15fa92a4 100644 --- a/src/api/posts/src/data/post.js +++ b/src/api/posts/src/data/post.js @@ -36,7 +36,7 @@ function ensureFeed(feed) { } class Post { - constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) { + constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) { // Use the post's guid as our unique identifier this.id = hash(guid); this.title = title; @@ -45,6 +45,7 @@ class Post { this.updated = ensureDate(dateUpdated, datePublished); this.url = postUrl; this.guid = guid; + this.type = type; if (!(feed instanceof Feed)) { throw new Error(`expected feed to be a Feed Object, got '${feed}'`); @@ -91,6 +92,7 @@ class Post { postData.updated, postData.url, postData.guid, + postData.type, feed ); await post.save(); @@ -116,6 +118,7 @@ class Post { data.updated, data.url, data.guid, + data.type, feed ); return post; diff --git a/src/api/posts/src/storage.js b/src/api/posts/src/storage.js index b181e96bdc..fe9eb6444e 100644 --- a/src/api/posts/src/storage.js +++ b/src/api/posts/src/storage.js @@ -154,6 +154,8 @@ module.exports = { post.url, 'guid', post.guid, + 'type', + post.type, 'feed', post.feed ) @@ -192,7 +194,7 @@ module.exports = { const key = createPostKey(id); await redis .multi() - .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed') + .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed') .zrem(postsKey, id) .exec(); }, diff --git a/src/backend/data/post.js b/src/backend/data/post.js index 933589d95b..c6546a042f 100644 --- a/src/backend/data/post.js +++ b/src/backend/data/post.js @@ -39,8 +39,27 @@ function ensureFeed(feed) { return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed); } +/** + * Determine the type of the post depending on the given article. + * The possible types are 'video' or 'blogpost'. + * + * @param {*} article + * @returns a String representing the type + */ +function determinePostType(article) { + const associatedLink = new URL(article.link); + + if (associatedLink.hostname.includes('youtube.com')) { + return 'video'; + } else { + // Assume that we are dealing with a blogpost if we + // are not dealing with videos + return 'blogpost'; + } +} + class Post { - constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) { + constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) { // Use the post's guid as our unique identifier this.id = hash(guid); this.title = title; @@ -49,6 +68,7 @@ class Post { this.updated = ensureDate(dateUpdated, datePublished); this.url = postUrl; this.guid = guid; + this.type = type; // We expect to get a real Feed vs. a feed id if (!(feed instanceof Feed)) { @@ -150,6 +170,7 @@ class Post { // link is the url to the post article.link, article.guid, + determinePostType(article), feed ); await Promise.all([post.save(), indexPost(post)]); @@ -197,6 +218,7 @@ class Post { data.updated, data.url, data.guid, + data.type, feed ); return post; diff --git a/src/backend/feed/processor.js b/src/backend/feed/processor.js index 0f9860ec7e..64f5c3f857 100644 --- a/src/backend/feed/processor.js +++ b/src/backend/feed/processor.js @@ -106,6 +106,10 @@ function articlesToPosts(articles, feed) { return Promise.all( articles.map(async (article) => { try { + if (article.mediaGroup) { + article.content = article.mediaGroup['media:description']; + } + await Post.createFromArticle(article, feed); } catch (error) { // If this is just some missing data, ignore the post, otherwise throw. @@ -184,22 +188,28 @@ module.exports = async function processor(job) { ['pubDate', 'pubdate'], ['creator', 'author'], ['content:encoded', 'contentEncoded'], + ['updated', 'date'], + ['id', 'guid'], + ['media:group', 'mediaGroup'], + ['published', 'pubdate'], ], }, }, feed ) ); - const articles = await parser.parseURL(feed.url); + + const feedXml = await parser.parseURL(feed.url); + // Transform the list of articles to a list of Post objects - await articlesToPosts(articles.items, feed); + await articlesToPosts(feedXml.items, feed); // Version info for this feed changed, so update the database feed.etag = feed.etag || info.etag; feed.lastModified = feed.lastModified || info.lastModified; // If feed.link is empty or there are blog posts - if (!feed.link && articles.items.length) { - feed.link = articles?.link || null; + if (!feed.link && feedXml.items.length) { + feed.link = feedXml?.link || null; } await feed.save(); } catch (error) { diff --git a/src/backend/index.js b/src/backend/index.js index 909dae295a..bb0b428f56 100644 --- a/src/backend/index.js +++ b/src/backend/index.js @@ -77,7 +77,14 @@ async function processAllFeeds() { // Get an Array of Feed objects from the wiki feed list and Redis const [all, wiki] = await Promise.all([Feed.all(), getWikiFeeds()]); // Process these feeds into the database and feed queue - await processFeeds([...all, ...wiki]); + await processFeeds( + /*[...all, ...wiki]*/ [ + { + url: 'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A', + author: 'David Humphrey', + }, + ] + ); } catch (err) { logger.error({ err }, 'Error queuing feeds'); } diff --git a/src/backend/utils/storage.js b/src/backend/utils/storage.js index fc32450311..f560d16678 100644 --- a/src/backend/utils/storage.js +++ b/src/backend/utils/storage.js @@ -120,6 +120,8 @@ module.exports = { post.url, 'guid', post.guid, + 'type', + post.type, 'feed', post.feed ) @@ -158,7 +160,7 @@ module.exports = { const key = createPostKey(id); await redis .multi() - .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed') + .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed') .zrem(postsKey, id) .exec(); },