diff --git a/src/api/posts/src/data/post.js b/src/api/posts/src/data/post.js index 656ca81ce8..9a817568c1 100644 --- a/src/api/posts/src/data/post.js +++ b/src/api/posts/src/data/post.js @@ -35,8 +35,30 @@ function ensureFeed(feed) { return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed); } +/** + * Determine the type of the post depending on the given URL. + * The possible types are 'video' or 'blogpost'. + * + * @param {*} url + * @returns a String representing the type + */ +function determinePostType(url) { + try { + const associatedLink = new URL(url); + + if (associatedLink.hostname.includes('youtube.com')) { + return 'video'; + } + // Assume that we are dealing with a blogpost if we + // are not dealing with videos + return 'blogpost'; + } catch { + return 'blogpost'; + } +} + class Post { - constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) { + constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) { // Use the post's guid as our unique identifier this.id = hash(guid); this.title = title; @@ -46,7 +68,7 @@ class Post { // create an absolute url if postURL is relative this.url = new URL(postUrl, feed.url).href; this.guid = guid; - this.type = type; + this.type = determinePostType(this.url); if (!(feed instanceof Feed)) { throw new Error(`expected feed to be a Feed Object, got '${feed}'`); @@ -93,7 +115,6 @@ class Post { postData.updated, postData.url, postData.guid, - postData.type, feed ); await post.save(); @@ -119,7 +140,6 @@ class Post { data.updated, data.url, data.guid, - data.type, feed ); return post; diff --git a/src/api/posts/src/storage.js b/src/api/posts/src/storage.js index e5d454e4f2..c3eb409aa2 100644 --- a/src/api/posts/src/storage.js +++ b/src/api/posts/src/storage.js @@ -165,8 +165,6 @@ module.exports = { post.url, 'guid', post.guid, - 'type', - post.type, 'feed', post.feed ) @@ -205,7 +203,7 @@ module.exports = { const key = createPostKey(id); await redis .multi() - .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed') + .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed') .zrem(postsKey, id) .exec(); }, diff --git a/src/backend/data/post.js b/src/backend/data/post.js index 780c00ec86..1d5d45a698 100644 --- a/src/backend/data/post.js +++ b/src/backend/data/post.js @@ -39,27 +39,8 @@ function ensureFeed(feed) { return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed); } -/** - * Determine the type of the post depending on the given article. - * The possible types are 'video' or 'blogpost'. - * - * @param {*} article - * @returns a String representing the type - */ -function determinePostType(article) { - const associatedLink = new URL(article.link); - - if (associatedLink.hostname.includes('youtube.com')) { - return 'video'; - } else { - // Assume that we are dealing with a blogpost if we - // are not dealing with videos - return 'blogpost'; - } -} - class Post { - constructor(title, html, datePublished, dateUpdated, postUrl, guid, type, feed) { + constructor(title, html, datePublished, dateUpdated, postUrl, guid, feed) { // Use the post's guid as our unique identifier this.id = hash(guid); this.title = title; @@ -69,7 +50,6 @@ class Post { // create an absolute url if postURL is relative this.url = new URL(postUrl, feed.url).href; this.guid = guid; - this.type = type; // We expect to get a real Feed vs. a feed id if (!(feed instanceof Feed)) { @@ -116,6 +96,8 @@ class Post { if (article.contentEncoded) article.content = article.contentEncoded; + if (article.mediaGroup) article.content = article.mediaGroup['media:description']; + // A valid RSS/Atom feed can have missing fields that we care about. // Keep track of any that are missing, and throw if necessary. const missing = []; @@ -171,7 +153,6 @@ class Post { // link is the url to the post article.link, article.guid, - determinePostType(article), feed ); await Promise.all([post.save(), indexPost(post)]); @@ -219,7 +200,6 @@ class Post { data.updated, data.url, data.guid, - data.type, feed ); return post; diff --git a/src/backend/feed/processor.js b/src/backend/feed/processor.js index 64f5c3f857..3df3fa1730 100644 --- a/src/backend/feed/processor.js +++ b/src/backend/feed/processor.js @@ -106,10 +106,6 @@ function articlesToPosts(articles, feed) { return Promise.all( articles.map(async (article) => { try { - if (article.mediaGroup) { - article.content = article.mediaGroup['media:description']; - } - await Post.createFromArticle(article, feed); } catch (error) { // If this is just some missing data, ignore the post, otherwise throw. diff --git a/src/backend/index.js b/src/backend/index.js index bb0b428f56..909dae295a 100644 --- a/src/backend/index.js +++ b/src/backend/index.js @@ -77,14 +77,7 @@ async function processAllFeeds() { // Get an Array of Feed objects from the wiki feed list and Redis const [all, wiki] = await Promise.all([Feed.all(), getWikiFeeds()]); // Process these feeds into the database and feed queue - await processFeeds( - /*[...all, ...wiki]*/ [ - { - url: 'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A', - author: 'David Humphrey', - }, - ] - ); + await processFeeds([...all, ...wiki]); } catch (err) { logger.error({ err }, 'Error queuing feeds'); } diff --git a/src/backend/utils/storage.js b/src/backend/utils/storage.js index 79ad045d3e..31dde56dcf 100644 --- a/src/backend/utils/storage.js +++ b/src/backend/utils/storage.js @@ -121,8 +121,6 @@ module.exports = { post.url, 'guid', post.guid, - 'type', - post.type, 'feed', post.feed ) @@ -161,7 +159,7 @@ module.exports = { const key = createPostKey(id); await redis .multi() - .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'type', 'feed') + .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed') .zrem(postsKey, id) .exec(); }, diff --git a/test/fixtures.js b/test/fixtures.js index a6c9c55356..19a06715c7 100644 --- a/test/fixtures.js +++ b/test/fixtures.js @@ -18,6 +18,12 @@ const getRealWorldRssUri = () => 'https://blog.humphd.org/tag/seneca/rss/'; const getRealWorldRssBody = () => fs.readFileSync(path.join(__dirname, './test_files/blog.humphd.org.rss')); +// Use David Humphrey's channel for a realistic test case of YouTube channel +const getRealWorldYouTubeFeedUri = () => + 'https://www.youtube.com/feeds/videos.xml?channel_id=UCqaMbMDf01BLttof1lHAo2A'; +const getRealWorldYouTubeFeedBody = () => + fs.readFileSync(path.join(__dirname, './test_files/humphd-yt-channel.xml')); + // Portion of https://www.feedforall.com/sample.xml const getValidFeedBody = () => ` @@ -131,6 +137,7 @@ exports.getAtomUri = getAtomUri; exports.getRssUri = getRssUri; exports.getHtmlUri = getHtmlUri; exports.getRealWorldRssUri = getRealWorldRssUri; +exports.getRealWorldYouTubeFeedUri = getRealWorldYouTubeFeedUri; exports.stripProtocol = stripProtocol; exports.getInvalidDescription = getInvalidDescription; @@ -162,4 +169,14 @@ exports.nockRealWorldRssResponse = function (headers = {}) { nockResponse(getRealWorldRssUri(), getRealWorldRssBody(), 200, 'application/rss+xml', headers); }; +exports.nockRealWorldYouTubeFeedResponse = function (headers = {}) { + nockResponse( + getRealWorldYouTubeFeedUri(), + getRealWorldYouTubeFeedBody(), + 200, + 'application/rss+xml', + headers + ); +}; + exports.createMockJobObjectFromFeedId = (id) => ({ data: { id } }); diff --git a/test/post.test.js b/test/post.test.js index b9d295e061..0a1c96106b 100644 --- a/test/post.test.js +++ b/test/post.test.js @@ -6,12 +6,18 @@ const parse = new Parser({ ['pubDate', 'pubdate'], ['creator', 'author'], ['content:encoded', 'contentEncoded'], + ['updated', 'date'], + ['id', 'guid'], + ['media:group', 'mediaGroup'], + ['published', 'pubdate'], ], }, }); const { nockRealWorldRssResponse, + nockRealWorldYouTubeFeedResponse, + getRealWorldYouTubeFeedUri, getRealWorldRssUri, getInvalidDescription, } = require('./fixtures'); @@ -175,7 +181,7 @@ describe('Post data class tests', () => { expect(result).toBe(null); }); - describe('Post.createFromArticle() tests', () => { + describe('Post.createFromArticle() with blog feeds tests', () => { let articles; beforeEach(async () => { nockRealWorldRssResponse(); @@ -282,4 +288,23 @@ describe('Post data class tests', () => { await expect(Post.createFromArticle(article, feed)).rejects.toThrow(); }); }); + + describe('Post.createFromArticle() with youtube feeds tests', () => { + let articles; + beforeEach(async () => { + nockRealWorldYouTubeFeedResponse(); + articles = await parse.parseURL(getRealWorldYouTubeFeedUri()); + expect(Array.isArray(articles.items)).toBe(true); + expect(articles.items.length).toBe(15); + }); + + test('Post.createFromArticle() should create Post with YouTube video article', async () => { + const article = articles.items[0]; + const id = await Post.createFromArticle(article, feed); + const videoPost = await Post.byId(id); + + expect(videoPost.title).toBe('DPS909 OSD600 Week 03 - Fixing a Bug in the Azure JS SDK'); + expect(videoPost.url).toBe('https://www.youtube.com/watch?v=mNuHA7vH6Wc'); + }); + }); }); diff --git a/test/test_files/humphd-yt-channel.xml b/test/test_files/humphd-yt-channel.xml new file mode 100644 index 0000000000..de3184fb7b --- /dev/null +++ b/test/test_files/humphd-yt-channel.xml @@ -0,0 +1,360 @@ + + + + yt:channel:UCqaMbMDf01BLttof1lHAo2A + UCqaMbMDf01BLttof1lHAo2A + David Humphrey + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2015-07-11T20:00:29+00:00 + + yt:video:mNuHA7vH6Wc + mNuHA7vH6Wc + UCqaMbMDf01BLttof1lHAo2A + DPS909 OSD600 Week 03 - Fixing a Bug in the Azure JS SDK + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-09-24T19:04:25+00:00 + 2021-10-27T10:46:10+00:00 + + DPS909 OSD600 Week 03 - Fixing a Bug in the Azure JS SDK + + + Walkthrough and discussion of fixing a bug in https://github.com/Azure/azure-sdk-for-js. Issue at https://github.com/Azure/azure-sdk-for-js/issues/15772. PR at https://github.com/Azure/azure-sdk-for-js/pull/17820. + + + + + + + + yt:video:GUXjyPp433M + GUXjyPp433M + UCqaMbMDf01BLttof1lHAo2A + DPS909/OSD600 Fall 2021 Week 01 Part 2 - Week 1 Overview + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-09-06T12:00:06+00:00 + 2021-09-16T13:20:23+00:00 + + DPS909/OSD600 Fall 2021 Week 01 Part 2 - Week 1 Overview + + + Overview of https://github.com/Seneca-CDOT/topics-in-open-source-2021 + + + + + + + + yt:video:rOBX6vRch7U + rOBX6vRch7U + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 12 WebSockets Part 1 - Intro and Writing the Server + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-04-04T13:00:30+00:00 + 2021-11-06T04:28:19+00:00 + + WEB422 Week 12 WebSockets Part 1 - Intro and Writing the Server + + + Introduction to Web Sockets and writing a Web Socket Server for an Emoji Chat App. Code at https://github.com/humphd/web422-week12 + + + + + + + + yt:video:lU9db7Dd95I + lU9db7Dd95I + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 12 WebSockets Part 2 - Writing the React Front-End + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-04-04T13:00:30+00:00 + 2021-04-05T23:01:11+00:00 + + WEB422 Week 12 WebSockets Part 2 - Writing the React Front-End + + + Creating the React Front-End for our Emoji Chat App. Code at https://github.com/humphd/web422-week12 + + + + + + + + yt:video:motlCbDr6c4 + motlCbDr6c4 + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 05 Part 4 - Deploying React Apps + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-02-07T14:00:13+00:00 + 2021-03-11T03:07:39+00:00 + + WEB422 Week 05 Part 4 - Deploying React Apps + + + Deploying a Create-React-App (https://create-react-app.dev/docs/deployment) to various services: GitHub Pages (https://pages.github.com/), Vercel (https://vercel.com/), Netlify (https://www.netlify.com/), Begin (https://begin.com/). + +Example deploy of (https://github.com/humphd/web422-week-05-artsy) to Vercel https://web422-week-05-artsy.vercel.app/ and GitHub Pages https://humphd.github.io/web422-week-05-artsy/build/ + + + + + + + + yt:video:TaSAzbTltUo + TaSAzbTltUo + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 05 Part 3 - Using Formik, Joi, and Yup with React + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-02-07T14:00:07+00:00 + 2021-08-19T07:07:52+00:00 + + WEB422 Week 05 Part 3 - Using Formik, Joi, and Yup with React + + + Example of working with Form data in the browser with Formik, and server using Express, and how to validate the data using Joi and Yup on both ends. Code available at https://github.com/humphd/web422-week05-formik. + + + + + + + + yt:video:NPf-Y2Ek6a4 + NPf-Y2Ek6a4 + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 05 Part 1 - Intro to React Forms + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-02-07T14:00:06+00:00 + 2021-05-08T09:50:22+00:00 + + WEB422 Week 05 Part 1 - Intro to React Forms + + + Introduction to React Forms, discussion of code in notes https://web422.ca/notes/react-forms. See also the official React docs on forms https://reactjs.org/docs/forms.html + + + + + + + + yt:video:5i10iPnrfmw + 5i10iPnrfmw + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 05 Part 2 - Using a Form to Load API Data + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-02-07T14:00:01+00:00 + 2021-11-03T23:24:37+00:00 + + WEB422 Week 05 Part 2 - Using a Form to Load API Data + + + Building an interactive Form in React that uses the Art Institute of Chicago REST API (http://api.artic.edu/docs/). Code available at https://github.com/humphd/web422-week-05-artsy + + + + + + + + yt:video:zcJA5YBzJG8 + zcJA5YBzJG8 + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 04 Part 3 - Using Third Party React Components for Routing and UI + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-01-31T14:00:30+00:00 + 2021-03-11T03:10:12+00:00 + + WEB422 Week 04 Part 3 - Using Third Party React Components for Routing and UI + + + Discussion of React Routing using React Router (https://reactrouter.com/web) and how we can use 3rd Party React Components like https://react-bootstrap.github.io/ + + + + + + + + yt:video:hgew3p5RriY + hgew3p5RriY + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 04 Part 1 - React Events + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-01-31T14:00:12+00:00 + 2021-03-11T04:53:59+00:00 + + WEB422 Week 04 Part 1 - React Events + + + Discussion of React Events and how they related to DOM Events (notes: https://web422.ca/notes/react-events-and-data). Code from video available at https://gist.github.com/humphd/e353ab107e561c496bf9eec78fa8cac4 + + + + + + + + yt:video:dDMgZ7TfPaI + dDMgZ7TfPaI + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 04 Part 4 - GitHub API Example + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-01-31T14:00:09+00:00 + 2021-03-11T04:50:40+00:00 + + WEB422 Week 04 Part 4 - GitHub API Example + + + GitHub API Example with React Router, Bootstrap React, and useSWR 3rd party Components. Code available at https://github.com/humphd/web422-week04-github-example. + + + + + + + + yt:video:WdT_coWe4ms + WdT_coWe4ms + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 04 Part 2 - React Events and Data Loading + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-01-31T14:00:06+00:00 + 2021-10-23T12:22:30+00:00 + + WEB422 Week 04 Part 2 - React Events and Data Loading + + + Working with Events, Data Loading, and Conditional Rendering in React. Code available at https://github.com/humphd/web422-week04-events-data-loading + + + + + + + + yt:video:rx4KuxqD3CA + rx4KuxqD3CA + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 03 Part 2 - Modern JavaScript in React + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-01-24T14:00:31+00:00 + 2021-09-12T04:43:29+00:00 + + WEB422 Week 03 Part 2 - Modern JavaScript in React + + + Discussion of some of the newer syntax in JavaScript that will be used extensively in React. See https://reactjs.org/docs/getting-started.html#javascript-resources and https://developer.mozilla.org/en-US/docs/Web/JavaScript/A_re-introduction_to_JavaScript and + + + + + + + + yt:video:wC0TOPzrLTI + wC0TOPzrLTI + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 03 Part 1 - React Intro + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-01-24T14:00:13+00:00 + 2021-01-24T14:00:13+00:00 + + WEB422 Week 03 Part 1 - React Intro + + + Intro to React. Discussion of some of the main ideas and philosophies of building web applications with React. + + + + + + + + yt:video:n1A52kPfPtI + n1A52kPfPtI + UCqaMbMDf01BLttof1lHAo2A + WEB422 Week 03 Part 4 - Writing an App with React + + + David Humphrey + https://www.youtube.com/channel/UCqaMbMDf01BLttof1lHAo2A + + 2021-01-24T14:00:12+00:00 + 2021-01-28T21:28:06+00:00 + + WEB422 Week 03 Part 4 - Writing an App with React + + + Rewrite of the Week 1 reqres.in API front-end (https://github.com/humphd/web422-week01) in React. Code available at https://github.com/humphd/web422-week03-react. Live version at https://humphd.github.io/web422-week03-react/dist/ + + + + + + + \ No newline at end of file