Implement videos as a type of posts

YouTube videos are tracked like regular blogposts,
as they also come from a type of RSS feed. We make
some changes to adjust to the type of RSS response we
get so we can extract the necessary data.

src/api/posts/src/data/post.js

@@ -35,6 +35,25 @@ function ensureFeed(feed) {
   return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
 }
 
+/**
+ * @param {string} url
+ * @returns {"video" | "blogpost"} the post's type
+ */
+function determinePostType(url) {
+  try {
+    const associatedLink = new URL(url);
+
+    if (associatedLink.hostname.includes('youtube.com')) {
+      return 'video';
+    }
+    // Assume that we are dealing with a blogpost if we
+    // are not dealing with videos
+    return 'blogpost';
+  } catch {
+    return 'blogpost';
+  }
+}
+
 class Post {
   constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
     // Use the post's guid as our unique identifier
@@ -46,6 +65,7 @@ class Post {
     // create an absolute url if postURL is relative
     this.url = new URL(postUrl, feed.url).href;
     this.guid = guid;
+    this.type = determinePostType(this.url);
 
     if (!(feed instanceof Feed)) {
       throw new Error(`expected feed to be a Feed Object, got '${feed}'`);

src/api/posts/test/posts.test.js

@@ -82,6 +82,7 @@ describe('/posts', () => {
 
 describe('test /posts/:id responses', () => {
   const missingGuid = 'http://missing-guid';
+  const youtubeGuid = 'http://youtube.com';
   const randomGuid = 'http://random-guid';
 
   const feed1 = new Feed(
@@ -93,7 +94,16 @@ describe('test /posts/:id responses', () => {
     null
   );
 
-  beforeAll(() => Promise.resolve(addFeed(feed1)));
+  const youtubeFeed = new Feed(
+    'YouTube Author',
+    'http://youtube.com/feed/videos.xml',
+    'user',
+    'https://youtube.com/',
+    null,
+    null
+  );
+
+  beforeAll(() => Promise.all([addFeed(feed1), addFeed(youtubeFeed)]));
 
   const addedPost1 = new Post(
     'Post Title',
@@ -105,11 +115,23 @@ describe('test /posts/:id responses', () => {
     feed1
   );
 
-  beforeAll(() => Promise.resolve(addPost(addedPost1)));
+  const addedVideo1 = new Post(
+    'YouTube Video Title',
+    'YouTube Video Description',
+    new Date('2009-09-07T22:20:00.000Z'),
+    new Date('2009-09-07T22:23:00.000Z'),
+    'https://youtube.com/watch',
+    youtubeGuid,
+    youtubeFeed
+  );
+
+  beforeAll(() => Promise.all([addPost(addedPost1), addPost(addedVideo1)]));
 
   beforeAll(() => {
     feed1.save();
+    youtubeFeed.save();
     addedPost1.save();
+    addedVideo1.save();
   });
 
   test('A post with an id should be returned and match the id of a post from redis', async () => {
@@ -160,4 +182,22 @@ describe('test /posts/:id responses', () => {
     expect(res.status).toEqual(404);
     expect(res.get('Content-length')).toEqual('46');
   });
+
+  test('request a post with type equal to "blogpost"', async () => {
+    const res = await request(app).get(`/${addedPost1.id}`).set('Accept', 'application/json');
+    const post = await getPost(`${addedPost1.id}`);
+    expect(res.status).toEqual(200);
+    expect(res.get('Content-type')).toContain('application/json');
+    expect(res.body.id).toEqual(post.id);
+    expect(res.body.type).toEqual('blogpost');
+  });
+
+  test('request a post with type equal to "video"', async () => {
+    const res = await request(app).get(`/${addedVideo1.id}`).set('Accept', 'application/json');
+    const post = await getPost(`${addedVideo1.id}`);
+    expect(res.status).toEqual(200);
+    expect(res.get('Content-type')).toContain('application/json');
+    expect(res.body.id).toEqual(post.id);
+    expect(res.body.type).toEqual('video');
+  });
 });

src/backend/data/post.js

@@ -39,6 +39,25 @@ function ensureFeed(feed) {
   return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
 }
 
+/**
+ * @param {string} url
+ * @returns {"video" | "blogpost"} the post's type
+ */
+function determinePostType(url) {
+  try {
+    const associatedLink = new URL(url);
+
+    if (associatedLink.hostname.includes('youtube.com')) {
+      return 'video';
+    }
+    // Assume that we are dealing with a blogpost if we
+    // are not dealing with videos
+    return 'blogpost';
+  } catch {
+    return 'blogpost';
+  }
+}
+
 class Post {
   constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) {
     // Use the post's guid as our unique identifier
@@ -50,6 +69,7 @@ class Post {
     // create an absolute url if postURL is relative
     this.url = new URL(postUrl, feed.url).href;
     this.guid = guid;
+    this.type = determinePostType(this.url);
 
     // We expect to get a real Feed vs. a feed id
     if (!(feed instanceof Feed)) {
@@ -96,6 +116,8 @@ class Post {
 
     if (article.contentEncoded) article.content = article.contentEncoded;
 
+    if (article.mediaGroup) article.content = article.mediaGroup['media:description'];
+
     // A valid RSS/Atom feed can have missing fields that we care about.
     // Keep track of any that are missing, and throw if necessary.
     const missing = [];

src/backend/feed/processor.js

@@ -184,13 +184,19 @@ module.exports = async function processor(job) {
               ['pubDate', 'pubdate'],
               ['creator', 'author'],
               ['content:encoded', 'contentEncoded'],
+              ['updated', 'date'],
+              ['id', 'guid'],
+              ['media:group', 'mediaGroup'],
+              ['published', 'pubdate'],
             ],
           },
         },
         feed
       )
     );
+
     const articles = await parser.parseURL(feed.url);
+
     // Transform the list of articles to a list of Post objects
     await articlesToPosts(articles.items, feed);
 

test/fixtures.js

@@ -18,6 +18,12 @@ const getRealWorldRssUri = () => 'https://blog.humphd.org/tag/seneca/rss/';
 const getRealWorldRssBody = () =>
   fs.readFileSync(path.join(__dirname, './test_files/blog.humphd.org.rss'));
 
+// Use David Humphrey's channel for a realistic test case of YouTube channel
+const getRealWorldYouTubeFeedUri = () =>
+  'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A';
+const getRealWorldYouTubeFeedBody = () =>
+  fs.readFileSync(path.join(__dirname, './test_files/humphd-yt-channel.xml'));
+
 // Portion of https://www.feedforall.com/sample.xml
 const getValidFeedBody = () =>
   `
@@ -131,6 +137,7 @@ exports.getAtomUri = getAtomUri;
 exports.getRssUri = getRssUri;
 exports.getHtmlUri = getHtmlUri;
 exports.getRealWorldRssUri = getRealWorldRssUri;
+exports.getRealWorldYouTubeFeedUri = getRealWorldYouTubeFeedUri;
 exports.stripProtocol = stripProtocol;
 exports.getInvalidDescription = getInvalidDescription;
 
@@ -162,4 +169,14 @@ exports.nockRealWorldRssResponse = function (headers = {}) {
   nockResponse(getRealWorldRssUri(), getRealWorldRssBody(), 200, 'application/rss+xml', headers);
 };
 
+exports.nockRealWorldYouTubeFeedResponse = function (headers = {}) {
+  nockResponse(
+    getRealWorldYouTubeFeedUri(),
+    getRealWorldYouTubeFeedBody(),
+    200,
+    'application/rss+xml',
+    headers
+  );
+};
+
 exports.createMockJobObjectFromFeedId = (id) => ({ data: { id } });

test/post.test.js

@@ -6,12 +6,18 @@ const parse = new Parser({
       ['pubDate', 'pubdate'],
       ['creator', 'author'],
       ['content:encoded', 'contentEncoded'],
+      ['updated', 'date'],
+      ['id', 'guid'],
+      ['media:group', 'mediaGroup'],
+      ['published', 'pubdate'],
     ],
   },
 });
 
 const {
   nockRealWorldRssResponse,
+  nockRealWorldYouTubeFeedResponse,
+  getRealWorldYouTubeFeedUri,
   getRealWorldRssUri,
   getInvalidDescription,
 } = require('./fixtures');
@@ -32,6 +38,7 @@ describe('Post data class tests', () => {
     url: 'https://user.post.com/?post-id=123',
     guid: 'https://user.post.com/?post-id=123&guid',
     id: hash('https://user.post.com/?post-id=123&guid'),
+    type: 'blogpost',
   };
 
   beforeAll(async () => {
@@ -175,7 +182,7 @@ describe('Post data class tests', () => {
     expect(result).toBe(null);
   });
 
-  describe('Post.createFromArticle() tests', () => {
+  describe('Post.createFromArticle() with blog feeds tests', () => {
     let articles;
     beforeEach(async () => {
       nockRealWorldRssResponse();
@@ -282,4 +289,24 @@ describe('Post data class tests', () => {
       await expect(Post.createFromArticle(article, feed)).rejects.toThrow();
     });
   });
+
+  describe('Post.createFromArticle() with youtube feeds tests', () => {
+    let articles;
+    beforeEach(async () => {
+      nockRealWorldYouTubeFeedResponse();
+      articles = await parse.parseURL(getRealWorldYouTubeFeedUri());
+      expect(Array.isArray(articles.items)).toBe(true);
+      expect(articles.items.length).toBe(15);
+    });
+
+    test('Post.createFromArticle() should create Post with YouTube video article', async () => {
+      const article = articles.items[0];
+      const id = await Post.createFromArticle(article, feed);
+      const videoPost = await Post.byId(id);
+
+      expect(videoPost.title).toBe('DPS909 OSD600 Week 03 - Fixing a Bug in the Azure JS SDK');
+      expect(videoPost.url).toBe('https://www.youtube.com/watch?v=mNuHA7vH6Wc');
+      expect(videoPost.type).toBe('video');
+    });
+  });
 });