Fix and add tests

src/api/posts/src/data/post.js

@@ -35,8 +35,30 @@ function ensureFeed(feed) {
   return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
 }
 
+/**
+ * Determine the type of the post depending on the given URL.
+ * The possible types are 'video' or 'blogpost'.
+ *
+ * @param {*} url
+ * @returns a String representing the type
+ */
+function determinePostType(url) {
+  try {
+    const associatedLink = new URL(url);
+
+    if (associatedLink.hostname.includes('youtube.com')) {
+      return 'video';
+    }
+    // Assume that we are dealing with a blogpost if we
+    // are not dealing with videos
+    return 'blogpost';
+  } catch {
+    return 'blogpost';
+  }
+}
+
 class Post {
-  constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) {
+  constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
     // Use the post's guid as our unique identifier
     this.id = hash(guid);
     this.title = title;
@@ -46,7 +68,7 @@ class Post {
     // create an absolute url if postURL is relative
     this.url = new URL(postUrl, feed.url).href;
     this.guid = guid;
-    this.type = type;
+    this.type = determinePostType(this.url);
 
     if (!(feed instanceof Feed)) {
       throw new Error(`expected feed to be a Feed Object, got '${feed}'`);
@@ -93,7 +115,6 @@ class Post {
       postData.updated,
       postData.url,
       postData.guid,
-      postData.type,
       feed
     );
     await post.save();
@@ -119,7 +140,6 @@ class Post {
       data.updated,
       data.url,
       data.guid,
-      data.type,
       feed
     );
     return post;

src/api/posts/src/storage.js

@@ -165,8 +165,6 @@ module.exports = {
         post.url,
         'guid',
         post.guid,
-        'type',
-        post.type,
         'feed',
         post.feed
       )
@@ -205,7 +203,7 @@ module.exports = {
     const key = createPostKey(id);
     await redis
       .multi()
-      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed')
+      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed')
       .zrem(postsKey, id)
       .exec();
   },

src/api/posts/test/posts.test.js

@@ -82,6 +82,7 @@ describe('/posts', () => {
 
 describe('test /posts/:id responses', () => {
   const missingGuid = 'http://missing-guid';
+  const youtubeGuid = 'http://youtube.com';
   const randomGuid = 'http://random-guid';
 
   const feed1 = new Feed(
@@ -93,7 +94,16 @@ describe('test /posts/:id responses', () => {
     null
   );
 
-  beforeAll(() => Promise.resolve(addFeed(feed1)));
+  const youtubeFeed = new Feed(
+    'YouTube Author',
+    'http://youtube.com/feed/videos.xml',
+    'user',
+    'https://youtube.com/',
+    null,
+    null
+  );
+
+  beforeAll(() => Promise.all([addFeed(feed1), addFeed(youtubeFeed)]));
 
   const addedPost1 = new Post(
     'Post Title',
@@ -105,11 +115,23 @@ describe('test /posts/:id responses', () => {
     feed1
   );
 
-  beforeAll(() => Promise.resolve(addPost(addedPost1)));
+  const addedVideo1 = new Post(
+    'YouTube Video Title',
+    'YouTube Video Description',
+    new Date('2009-09-07T22:20:00.000Z'),
+    new Date('2009-09-07T22:23:00.000Z'),
+    'https://youtube.com/watch',
+    youtubeGuid,
+    youtubeFeed
+  );
+
+  beforeAll(() => Promise.all([addPost(addedPost1), addPost(addedVideo1)]));
 
   beforeAll(() => {
     feed1.save();
+    youtubeFeed.save();
     addedPost1.save();
+    addedVideo1.save();
   });
 
   test('A post with an id should be returned and match the id of a post from redis', async () => {
@@ -160,4 +182,22 @@ describe('test /posts/:id responses', () => {
     expect(res.status).toEqual(404);
     expect(res.get('Content-length')).toEqual('46');
   });
+
+  test('request a post with type equal to "blogpost"', async () => {
+    const res = await request(app).get(`/${addedPost1.id}`).set('Accept', 'application/json');
+    const post = await getPost(`${addedPost1.id}`);
+    expect(res.status).toEqual(200);
+    expect(res.get('Content-type')).toContain('application/json');
+    expect(res.body.id).toEqual(post.id);
+    expect(res.body.type).toEqual('blogpost');
+  });
+
+  test('request a post with type equal to "video"', async () => {
+    const res = await request(app).get(`/${addedVideo1.id}`).set('Accept', 'application/json');
+    const post = await getPost(`${addedVideo1.id}`);
+    expect(res.status).toEqual(200);
+    expect(res.get('Content-type')).toContain('application/json');
+    expect(res.body.id).toEqual(post.id);
+    expect(res.body.type).toEqual('video');
+  });
 });

src/backend/data/post.js

@@ -39,27 +39,8 @@ function ensureFeed(feed) {
   return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
 }
 
-/**
- * Determine the type of the post depending on the given article.
- * The possible types are 'video' or 'blogpost'.
- *
- * @param {*} article
- * @returns a String representing the type
- */
-function determinePostType(article) {
-  const associatedLink = new URL(article.link);
-
-  if (associatedLink.hostname.includes('youtube.com')) {
-    return 'video';
-  } else {
-    // Assume that we are dealing with a blogpost if we
-    // are not dealing with videos
-    return 'blogpost';
-  }
-}
-
 class Post {
-  constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) {
+  constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
     // Use the post's guid as our unique identifier
     this.id = hash(guid);
     this.title = title;
@@ -69,7 +50,6 @@ class Post {
     // create an absolute url if postURL is relative
     this.url = new URL(postUrl, feed.url).href;
     this.guid = guid;
-    this.type = type;
 
     // We expect to get a real Feed vs. a feed id
     if (!(feed instanceof Feed)) {
@@ -116,6 +96,8 @@ class Post {
 
     if (article.contentEncoded) article.content = article.contentEncoded;
 
+    if (article.mediaGroup) article.content = article.mediaGroup['media:description'];
+
     // A valid RSS/Atom feed can have missing fields that we care about.
     // Keep track of any that are missing, and throw if necessary.
     const missing = [];
@@ -171,7 +153,6 @@ class Post {
       // link is the url to the post
       article.link,
       article.guid,
-      determinePostType(article),
       feed
     );
     await Promise.all([post.save(), indexPost(post)]);
@@ -219,7 +200,6 @@ class Post {
       data.updated,
       data.url,
       data.guid,
-      data.type,
       feed
     );
     return post;

src/backend/feed/processor.js

@@ -106,10 +106,6 @@ function articlesToPosts(articles, feed) {
   return Promise.all(
     articles.map(async (article) => {
       try {
-        if (article.mediaGroup) {
-          article.content = article.mediaGroup['media:description'];
-        }
-
         await Post.createFromArticle(article, feed);
       } catch (error) {
         // If this is just some missing data, ignore the post, otherwise throw.

src/backend/index.js

@@ -77,14 +77,7 @@ async function processAllFeeds() {
     // Get an Array of Feed objects from the wiki feed list and Redis
     const [all, wiki] = await Promise.all([Feed.all(), getWikiFeeds()]);
     // Process these feeds into the database and feed queue
-    await processFeeds(
-      /*[...all, ...wiki]*/ [
-        {
-          url: 'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A',
-          author: 'David Humphrey',
-        },
-      ]
-    );
+    await processFeeds([...all, ...wiki]);
   } catch (err) {
     logger.error({ err }, 'Error queuing feeds');
   }

src/backend/utils/storage.js

@@ -121,8 +121,6 @@ module.exports = {
         post.url,
         'guid',
         post.guid,
-        'type',
-        post.type,
         'feed',
         post.feed
       )
@@ -161,7 +159,7 @@ module.exports = {
     const key = createPostKey(id);
     await redis
       .multi()
-      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed')
+      .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed')
       .zrem(postsKey, id)
       .exec();
   },

test/fixtures.js

@@ -18,6 +18,12 @@ const getRealWorldRssUri = () => 'https://blog.humphd.org/tag/seneca/rss/';
 const getRealWorldRssBody = () =>
   fs.readFileSync(path.join(__dirname, './test_files/blog.humphd.org.rss'));
 
+// Use David Humphrey's channel for a realistic test case of YouTube channel
+const getRealWorldYouTubeFeedUri = () =>
+  'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A';
+const getRealWorldYouTubeFeedBody = () =>
+  fs.readFileSync(path.join(__dirname, './test_files/humphd-yt-channel.xml'));
+
 // Portion of https://www.feedforall.com/sample.xml
 const getValidFeedBody = () =>
   `
@@ -131,6 +137,7 @@ exports.getAtomUri = getAtomUri;
 exports.getRssUri = getRssUri;
 exports.getHtmlUri = getHtmlUri;
 exports.getRealWorldRssUri = getRealWorldRssUri;
+exports.getRealWorldYouTubeFeedUri = getRealWorldYouTubeFeedUri;
 exports.stripProtocol = stripProtocol;
 exports.getInvalidDescription = getInvalidDescription;
 
@@ -162,4 +169,14 @@ exports.nockRealWorldRssResponse = function (headers = {}) {
   nockResponse(getRealWorldRssUri(), getRealWorldRssBody(), 200, 'application/rss+xml', headers);
 };
 
+exports.nockRealWorldYouTubeFeedResponse = function (headers = {}) {
+  nockResponse(
+    getRealWorldYouTubeFeedUri(),
+    getRealWorldYouTubeFeedBody(),
+    200,
+    'application/rss+xml',
+    headers
+  );
+};
+
 exports.createMockJobObjectFromFeedId = (id) => ({ data: { id } });

test/post.test.js

@@ -6,12 +6,18 @@ const parse = new Parser({
       ['pubDate', 'pubdate'],
       ['creator', 'author'],
       ['content:encoded', 'contentEncoded'],
+      ['updated', 'date'],
+      ['id', 'guid'],
+      ['media:group', 'mediaGroup'],
+      ['published', 'pubdate'],
     ],
   },
 });
 
 const {
   nockRealWorldRssResponse,
+  nockRealWorldYouTubeFeedResponse,
+  getRealWorldYouTubeFeedUri,
   getRealWorldRssUri,
   getInvalidDescription,
 } = require('./fixtures');
@@ -175,7 +181,7 @@ describe('Post data class tests', () => {
     expect(result).toBe(null);
   });
 
-  describe('Post.createFromArticle() tests', () => {
+  describe('Post.createFromArticle() with blog feeds tests', () => {
     let articles;
     beforeEach(async () => {
       nockRealWorldRssResponse();
@@ -282,4 +288,23 @@ describe('Post data class tests', () => {
       await expect(Post.createFromArticle(article, feed)).rejects.toThrow();
     });
   });
+
+  describe('Post.createFromArticle() with youtube feeds tests', () => {
+    let articles;
+    beforeEach(async () => {
+      nockRealWorldYouTubeFeedResponse();
+      articles = await parse.parseURL(getRealWorldYouTubeFeedUri());
+      expect(Array.isArray(articles.items)).toBe(true);
+      expect(articles.items.length).toBe(15);
+    });
+
+    test('Post.createFromArticle() should create Post with YouTube video article', async () => {
+      const article = articles.items[0];
+      const id = await Post.createFromArticle(article, feed);
+      const videoPost = await Post.byId(id);
+
+      expect(videoPost.title).toBe('DPS909 OSD600 Week 03 - Fixing a Bug in the Azure JS SDK');
+      expect(videoPost.url).toBe('https://www.youtube.com/watch?v=mNuHA7vH6Wc');
+    });
+  });
 });