Implement videos as a type of posts

src/api/posts/src/data/post.js

@@ -36,7 +36,7 @@ function ensureFeed(feed) {
 }
 
 class Post {
-  constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
+  constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) {
     // Use the post's guid as our unique identifier
     this.id = hash(guid);
     this.title = title;
@@ -45,6 +45,7 @@ class Post {
     this.updated = ensureDate(dateUpdated, datePublished);
     this.url = postUrl;
     this.guid = guid;
+    this.type = type;
 
     if (!(feed instanceof Feed)) {
       throw new Error(`expected feed to be a Feed Object, got '${feed}'`);
@@ -91,6 +92,7 @@ class Post {
       postData.updated,
       postData.url,
       postData.guid,
+      postData.type,
       feed
     );
     await post.save();
@@ -116,6 +118,7 @@ class Post {
       data.updated,
       data.url,
       data.guid,
+      data.type,
       feed
     );
     return post;

src/api/posts/src/storage.js

@@ -154,6 +154,8 @@ module.exports = {
         post.url,
         'guid',
         post.guid,
+        'type',
+        post.type,
         'feed',
         post.feed
       )
@@ -192,7 +194,7 @@ module.exports = {
     const key = createPostKey(id);
     await redis
       .multi()
-      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed')
+      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed')
       .zrem(postsKey, id)
       .exec();
   },

src/backend/data/post.js

@@ -39,8 +39,27 @@ function ensureFeed(feed) {
   return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
 }
 
+/**
+ * Determine the type of the post depending on the given article.
+ * The possible types are 'video' or 'blogpost'.
+ *
+ * @param {*} article
+ * @returns a String representing the type
+ */
+function determinePostType(article) {
+  const associatedLink = new URL(article.link);
+
+  if (associatedLink.hostname.includes('youtube.com')) {
+    return 'video';
+  } else {
+    // Assume that we are dealing with a blogpost if we
+    // are not dealing with videos
+    return 'blogpost';
+  }
+}
+
 class Post {
-  constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
+  constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) {
     // Use the post's guid as our unique identifier
     this.id = hash(guid);
     this.title = title;
@@ -49,6 +68,7 @@ class Post {
     this.updated = ensureDate(dateUpdated, datePublished);
     this.url = postUrl;
     this.guid = guid;
+    this.type = type;
 
     // We expect to get a real Feed vs. a feed id
     if (!(feed instanceof Feed)) {
@@ -150,6 +170,7 @@ class Post {
       // link is the url to the post
       article.link,
       article.guid,
+      determinePostType(article),
       feed
     );
     await Promise.all([post.save(), indexPost(post)]);
@@ -197,6 +218,7 @@ class Post {
       data.updated,
       data.url,
       data.guid,
+      data.type,
       feed
     );
     return post;

src/backend/feed/processor.js

@@ -106,6 +106,10 @@ function articlesToPosts(articles, feed) {
   return Promise.all(
     articles.map(async (article) => {
       try {
+        if (article.mediaGroup) {
+          article.content = article.mediaGroup['media:description'];
+        }
+
         await Post.createFromArticle(article, feed);
       } catch (error) {
         // If this is just some missing data, ignore the post, otherwise throw.
@@ -184,22 +188,28 @@ module.exports = async function processor(job) {
               ['pubDate', 'pubdate'],
               ['creator', 'author'],
               ['content:encoded', 'contentEncoded'],
+              ['updated', 'date'],
+              ['id', 'guid'],
+              ['media:group', 'mediaGroup'],
+              ['published', 'pubdate'],
             ],
           },
         },
         feed
       )
     );
-    const articles = await parser.parseURL(feed.url);
+
+    const feedXml = await parser.parseURL(feed.url);
+
     // Transform the list of articles to a list of Post objects
-    await articlesToPosts(articles.items, feed);
+    await articlesToPosts(feedXml.items, feed);
 
     // Version info for this feed changed, so update the database
     feed.etag = feed.etag || info.etag;
     feed.lastModified = feed.lastModified || info.lastModified;
     // If feed.link is empty or there are blog posts
-    if (!feed.link && articles.items.length) {
-      feed.link = articles?.link || null;
+    if (!feed.link && feedXml.items.length) {
+      feed.link = feedXml?.link || null;
     }
     await feed.save();
   } catch (error) {

src/backend/index.js

@@ -77,7 +77,14 @@ async function processAllFeeds() {
     // Get an Array of Feed objects from the wiki feed list and Redis
     const [all, wiki] = await Promise.all([Feed.all(), getWikiFeeds()]);
     // Process these feeds into the database and feed queue
-    await processFeeds([...all, ...wiki]);
+    await processFeeds(
+      /*[...all, ...wiki]*/ [
+        {
+          url: 'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A',
+          author: 'David Humphrey',
+        },
+      ]
+    );
   } catch (err) {
     logger.error({ err }, 'Error queuing feeds');
   }

src/backend/utils/storage.js

@@ -120,6 +120,8 @@ module.exports = {
         post.url,
         'guid',
         post.guid,
+        'type',
+        post.type,
         'feed',
         post.feed
       )
@@ -158,7 +160,7 @@ module.exports = {
     const key = createPostKey(id);
     await redis
       .multi()
-      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed')
+      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed')
       .zrem(postsKey, id)
       .exec();
   },