Fix tests

src/api/posts/src/data/post.js

@@ -35,8 +35,30 @@ function ensureFeed(feed) {
   return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
 }
 
+/**
+ * Determine the type of the post depending on the given URL.
+ * The possible types are 'video' or 'blogpost'.
+ *
+ * @param {*} url
+ * @returns a String representing the type
+ */
+function determinePostType(url) {
+  try {
+    const associatedLink = new URL(url);
+
+    if (associatedLink.hostname.includes('youtube.com')) {
+      return 'video';
+    }
+    // Assume that we are dealing with a blogpost if we
+    // are not dealing with videos
+    return 'blogpost';
+  } catch {
+    return 'blogpost';
+  }
+}
+
 class Post {
-  constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) {
+  constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
     // Use the post's guid as our unique identifier
     this.id = hash(guid);
     this.title = title;
@@ -45,7 +67,7 @@ class Post {
     this.updated = ensureDate(dateUpdated, datePublished);
     this.url = postUrl;
     this.guid = guid;
-    this.type = type;
+    this.type = determinePostType(this.url);
 
     if (!(feed instanceof Feed)) {
       throw new Error(`expected feed to be a Feed Object, got '${feed}'`);
@@ -92,7 +114,6 @@ class Post {
       postData.updated,
       postData.url,
       postData.guid,
-      postData.type,
       feed
     );
     await post.save();
@@ -118,7 +139,6 @@ class Post {
       data.updated,
       data.url,
       data.guid,
-      data.type,
       feed
     );
     return post;

src/api/posts/src/storage.js

@@ -164,8 +164,6 @@ module.exports = {
         post.url,
         'guid',
         post.guid,
-        'type',
-        post.type,
         'feed',
         post.feed
       )
@@ -204,7 +202,7 @@ module.exports = {
     const key = createPostKey(id);
     await redis
       .multi()
-      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed')
+      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed')
       .zrem(postsKey, id)
       .exec();
   },

src/backend/data/post.js

@@ -39,27 +39,8 @@ function ensureFeed(feed) {
   return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
 }
 
-/**
- * Determine the type of the post depending on the given article.
- * The possible types are 'video' or 'blogpost'.
- *
- * @param {*} article
- * @returns a String representing the type
- */
-function determinePostType(article) {
-  const associatedLink = new URL(article.link);
-
-  if (associatedLink.hostname.includes('youtube.com')) {
-    return 'video';
-  } else {
-    // Assume that we are dealing with a blogpost if we
-    // are not dealing with videos
-    return 'blogpost';
-  }
-}
-
 class Post {
-  constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) {
+  constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
     // Use the post's guid as our unique identifier
     this.id = hash(guid);
     this.title = title;
@@ -68,7 +49,6 @@ class Post {
     this.updated = ensureDate(dateUpdated, datePublished);
     this.url = postUrl;
     this.guid = guid;
-    this.type = type;
 
     // We expect to get a real Feed vs. a feed id
     if (!(feed instanceof Feed)) {
@@ -115,6 +95,8 @@ class Post {
 
     if (article.contentEncoded) article.content = article.contentEncoded;
 
+    if (article.mediaGroup) article.content = article.mediaGroup['media:description'];
+
     // A valid RSS/Atom feed can have missing fields that we care about.
     // Keep track of any that are missing, and throw if necessary.
     const missing = [];
@@ -170,7 +152,6 @@ class Post {
       // link is the url to the post
       article.link,
       article.guid,
-      determinePostType(article),
       feed
     );
     await Promise.all([post.save(), indexPost(post)]);
@@ -218,7 +199,6 @@ class Post {
       data.updated,
       data.url,
       data.guid,
-      data.type,
       feed
     );
     return post;

src/backend/feed/processor.js

@@ -106,10 +106,6 @@ function articlesToPosts(articles, feed) {
   return Promise.all(
     articles.map(async (article) => {
       try {
-        if (article.mediaGroup) {
-          article.content = article.mediaGroup['media:description'];
-        }
-
         await Post.createFromArticle(article, feed);
       } catch (error) {
         // If this is just some missing data, ignore the post, otherwise throw.

src/backend/index.js

@@ -77,14 +77,7 @@ async function processAllFeeds() {
     // Get an Array of Feed objects from the wiki feed list and Redis
     const [all, wiki] = await Promise.all([Feed.all(), getWikiFeeds()]);
     // Process these feeds into the database and feed queue
-    await processFeeds(
-      /*[...all, ...wiki]*/ [
-        {
-          url: 'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A',
-          author: 'David Humphrey',
-        },
-      ]
-    );
+    await processFeeds([...all, ...wiki]);
   } catch (err) {
     logger.error({ err }, 'Error queuing feeds');
   }

src/backend/utils/storage.js

@@ -120,8 +120,6 @@ module.exports = {
         post.url,
         'guid',
         post.guid,
-        'type',
-        post.type,
         'feed',
         post.feed
       )
@@ -160,7 +158,7 @@ module.exports = {
     const key = createPostKey(id);
     await redis
       .multi()
-      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed')
+      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed')
       .zrem(postsKey, id)
       .exec();
   },

test/fixtures.js

@@ -18,6 +18,12 @@ const getRealWorldRssUri = () => 'https://blog.humphd.org/tag/seneca/rss/';
 const getRealWorldRssBody = () =>
   fs.readFileSync(path.join(__dirname, './test_files/blog.humphd.org.rss'));
 
+// Use David Humphrey's channel for a realistic test case of YouTube channel
+const getRealWorldYouTubeFeedUri = () =>
+  'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A';
+const getRealWorldYouTubeFeedBody = () =>
+  fs.readFileSync(path.join(__dirname, './test_files/humphd-yt-channel.xml'));
+
 // Portion of https://www.feedforall.com/sample.xml
 const getValidFeedBody = () =>
   `
@@ -131,6 +137,7 @@ exports.getAtomUri = getAtomUri;
 exports.getRssUri = getRssUri;
 exports.getHtmlUri = getHtmlUri;
 exports.getRealWorldRssUri = getRealWorldRssUri;
+exports.getRealWorldYouTubeFeedUri = getRealWorldYouTubeFeedUri;
 exports.stripProtocol = stripProtocol;
 exports.getInvalidDescription = getInvalidDescription;
 
@@ -162,4 +169,14 @@ exports.nockRealWorldRssResponse = function (headers = {}) {
   nockResponse(getRealWorldRssUri(), getRealWorldRssBody(), 200, 'application/rss+xml', headers);
 };
 
+exports.nockRealWorldYouTubeFeedResponse = function (headers = {}) {
+  nockResponse(
+    getRealWorldYouTubeFeedUri(),
+    getRealWorldYouTubeFeedBody(),
+    200,
+    'application/rss+xml',
+    headers
+  );
+};
+
 exports.createMockJobObjectFromFeedId = (id) => ({ data: { id } });

test/post.test.js

@@ -6,12 +6,18 @@ const parse = new Parser({
       ['pubDate', 'pubdate'],
       ['creator', 'author'],
       ['content:encoded', 'contentEncoded'],
+      ['updated', 'date'],
+      ['id', 'guid'],
+      ['media:group', 'mediaGroup'],
+      ['published', 'pubdate'],
     ],
   },
 });
 
 const {
   nockRealWorldRssResponse,
+  nockRealWorldYouTubeFeedResponse,
+  getRealWorldYouTubeFeedUri,
   getRealWorldRssUri,
   getInvalidDescription,
 } = require('./fixtures');
@@ -175,7 +181,7 @@ describe('Post data class tests', () => {
     expect(result).toBe(null);
   });
 
-  describe('Post.createFromArticle() tests', () => {
+  describe('Post.createFromArticle() with blog feeds tests', () => {
     let articles;
     beforeEach(async () => {
       nockRealWorldRssResponse();
@@ -282,4 +288,14 @@ describe('Post data class tests', () => {
       await expect(Post.createFromArticle(article, feed)).rejects.toThrow();
     });
   });
+
+  describe('Post.createFromArticle() with youtube feeds tests', () => {
+    let articles;
+    beforeEach(async () => {
+      nockRealWorldYouTubeFeedResponse();
+      articles = await parse.parseURL(getRealWorldYouTubeFeedUri());
+      expect(Array.isArray(articles.items)).toBe(true);
+      expect(articles.items.length).toBe(15);
+    });
+  });
 });