Fix and add tests

src/api/posts/test/posts.test.js

@@ -102,6 +102,7 @@ describe('test /posts/:id responses', () => {
     new Date('2009-09-07T22:23:00.000Z'),
     'url',
     missingGuid,
+    'blogpost',
     feed1
   );
 

src/backend/data/post.js

@@ -47,14 +47,17 @@ function ensureFeed(feed) {
  * @returns a String representing the type
  */
 function determinePostType(article) {
-  const associatedLink = new URL(article.link);
+  try {
+    const associatedLink = new URL(article.link);
 
-  if (associatedLink.hostname.includes('youtube.com')) {
-    return 'video';
-  } else {
+    if (associatedLink.hostname.includes('youtube.com')) {
+      return 'video';
+    }
     // Assume that we are dealing with a blogpost if we
     // are not dealing with videos
     return 'blogpost';
+  } catch {
+    return 'blogpost';
   }
 }
 
@@ -115,6 +118,8 @@ class Post {
 
     if (article.contentEncoded) article.content = article.contentEncoded;
 
+    if (article.mediaGroup) article.content = article.mediaGroup['media:description'];
+
     // A valid RSS/Atom feed can have missing fields that we care about.
     // Keep track of any that are missing, and throw if necessary.
     const missing = [];
@@ -193,6 +198,7 @@ class Post {
       postData.updated,
       postData.url,
       postData.guid,
+      postData.type,
       feed
     );
     await post.save();

src/backend/feed/processor.js

@@ -106,10 +106,6 @@ function articlesToPosts(articles, feed) {
   return Promise.all(
     articles.map(async (article) => {
       try {
-        if (article.mediaGroup) {
-          article.content = article.mediaGroup['media:description'];
-        }
-
         await Post.createFromArticle(article, feed);
       } catch (error) {
         // If this is just some missing data, ignore the post, otherwise throw.

src/backend/index.js

@@ -77,14 +77,7 @@ async function processAllFeeds() {
     // Get an Array of Feed objects from the wiki feed list and Redis
     const [all, wiki] = await Promise.all([Feed.all(), getWikiFeeds()]);
     // Process these feeds into the database and feed queue
-    await processFeeds(
-      /*[...all, ...wiki]*/ [
-        {
-          url: 'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A',
-          author: 'David Humphrey',
-        },
-      ]
-    );
+    await processFeeds([...all, ...wiki]);
   } catch (err) {
     logger.error({ err }, 'Error queuing feeds');
   }

test/fixtures.js

@@ -18,6 +18,12 @@ const getRealWorldRssUri = () => 'https://blog.humphd.org/tag/seneca/rss/';
 const getRealWorldRssBody = () =>
   fs.readFileSync(path.join(__dirname, './test_files/blog.humphd.org.rss'));
 
+// Use David Humphrey's channel for a realistic test case of YouTube channel
+const getRealWorldYouTubeFeedUri = () =>
+  'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A';
+const getRealWorldYouTubeFeedBody = () =>
+  fs.readFileSync(path.join(__dirname, './test_files/humphd-yt-channel.xml'));
+
 // Portion of https://www.feedforall.com/sample.xml
 const getValidFeedBody = () =>
   `
@@ -131,6 +137,7 @@ exports.getAtomUri = getAtomUri;
 exports.getRssUri = getRssUri;
 exports.getHtmlUri = getHtmlUri;
 exports.getRealWorldRssUri = getRealWorldRssUri;
+exports.getRealWorldYouTubeFeedUri = getRealWorldYouTubeFeedUri;
 exports.stripProtocol = stripProtocol;
 exports.getInvalidDescription = getInvalidDescription;
 
@@ -162,4 +169,14 @@ exports.nockRealWorldRssResponse = function (headers = {}) {
   nockResponse(getRealWorldRssUri(), getRealWorldRssBody(), 200, 'application/rss+xml', headers);
 };
 
+exports.nockRealWorldYouTubeFeedResponse = function (headers = {}) {
+  nockResponse(
+    getRealWorldYouTubeFeedUri(),
+    getRealWorldYouTubeFeedBody(),
+    200,
+    'application/rss+xml',
+    headers
+  );
+};
+
 exports.createMockJobObjectFromFeedId = (id) => ({ data: { id } });

test/post.test.js

@@ -6,12 +6,18 @@ const parse = new Parser({
       ['pubDate', 'pubdate'],
       ['creator', 'author'],
       ['content:encoded', 'contentEncoded'],
+      ['updated', 'date'],
+      ['id', 'guid'],
+      ['media:group', 'mediaGroup'],
+      ['published', 'pubdate'],
     ],
   },
 });
 
 const {
   nockRealWorldRssResponse,
+  nockRealWorldYouTubeFeedResponse,
+  getRealWorldYouTubeFeedUri,
   getRealWorldRssUri,
   getInvalidDescription,
 } = require('./fixtures');
@@ -32,6 +38,7 @@ describe('Post data class tests', () => {
     url: 'https://user.post.com/?post-id=123',
     guid: 'https://user.post.com/?post-id=123&guid',
     id: hash('https://user.post.com/?post-id=123&guid'),
+    type: 'blogpost',
   };
 
   beforeAll(async () => {
@@ -48,7 +55,16 @@ describe('Post data class tests', () => {
   const text = 'post text';
 
   const createPost = () =>
-    new Post(data.title, data.html, data.published, data.updated, data.url, data.guid, feed);
+    new Post(
+      data.title,
+      data.html,
+      data.published,
+      data.updated,
+      data.url,
+      data.guid,
+      data.type,
+      feed
+    );
 
   test('Post should be a function', () => {
     expect(typeof Post).toBe('function');
@@ -62,6 +78,7 @@ describe('Post data class tests', () => {
     expect(post.published).toEqual(data.published);
     expect(post.updated).toEqual(data.updated);
     expect(post.guid).toEqual(data.guid);
+    expect(post.type).toEqual(data.type);
     expect(post.feed).toEqual(feed);
   });
 
@@ -74,6 +91,7 @@ describe('Post data class tests', () => {
       'Thu, 20 Nov 2014 18:59:18 UTC',
       data.url,
       data.guid,
+      data.type,
       feed
     );
 
@@ -89,6 +107,7 @@ describe('Post data class tests', () => {
       new Date('Thu, 20 Nov 2014 18:59:18 UTC'),
       data.url,
       data.guid,
+      data.type,
       feed
     );
 
@@ -97,7 +116,16 @@ describe('Post data class tests', () => {
 
   test('Post constructor should throw if feed is missing or not a Feed instance', () => {
     const createPostWithFeed = (f) =>
-      new Post(data.title, data.html, data.published, data.updated, data.url, data.guid, f);
+      new Post(
+        data.title,
+        data.html,
+        data.published,
+        data.updated,
+        data.url,
+        data.guid,
+        data.type,
+        f
+      );
 
     expect(() => createPostWithFeed(null)).toThrow();
     expect(() => createPostWithFeed(undefined)).toThrow();
@@ -109,7 +137,16 @@ describe('Post data class tests', () => {
 
   test('Post constructor should throw if a string or date is not passed', () => {
     const createPostWithDates = (datePublished, dateUpdated) =>
-      new Post(data.title, data.html, datePublished, dateUpdated, data.url, data.guid, feed);
+      new Post(
+        data.title,
+        data.html,
+        datePublished,
+        dateUpdated,
+        data.url,
+        data.guid,
+        data.type,
+        feed
+      );
     expect(() =>
       createPostWithDates(
         new Date('Thu, 20 Nov 2014 18:59:18 UTC'),
@@ -175,7 +212,7 @@ describe('Post data class tests', () => {
     expect(result).toBe(null);
   });
 
-  describe('Post.createFromArticle() tests', () => {
+  describe('Post.createFromArticle() with blog feeds tests', () => {
     let articles;
     beforeEach(async () => {
       nockRealWorldRssResponse();
@@ -282,4 +319,22 @@ describe('Post data class tests', () => {
       await expect(Post.createFromArticle(article, feed)).rejects.toThrow();
     });
   });
+
+  describe('Post.createFromArticle() with youtube feeds tests', () => {
+    let articles;
+    beforeEach(async () => {
+      nockRealWorldYouTubeFeedResponse();
+      articles = await parse.parseURL(getRealWorldYouTubeFeedUri());
+      expect(Array.isArray(articles.items)).toBe(true);
+      expect(articles.items.length).toBe(15);
+    });
+
+    test('Post.createFromArticle() should create Post with type equal to video', async () => {
+      const article = articles.items[0];
+      const id = await Post.createFromArticle(article, feed);
+      const videoPost = await Post.byId(id);
+
+      expect(videoPost.type).toBe('video');
+    });
+  });
 });