diff --git a/config/env.development b/config/env.development
index 137caf5435..f6f514a423 100644
--- a/config/env.development
+++ b/config/env.development
@@ -155,6 +155,15 @@ PARSER_PORT=10000
# Parser Service URL
PARSER_URL=http://localhost/v1/parser
+################################################################################
+# Videos Service
+################################################################################
+
+# Video Service Port (default is 11234)
+VIDEOS_PORT=11234
+
+# Video Service URL
+VIDEOS_URL=http://localhost/v1/videos
################################################################################
# Planet Service
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index c9409e6a43..d9b53f0e57 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -263,6 +263,31 @@ services:
- PLANET_PORT
- POSTS_URL
+ # videos service
+ videos:
+ container_name: 'videos'
+ build:
+ context: ../src/api/videos
+ dockerfile: Dockerfile
+ environment:
+ - NODE_ENV
+ - VIDEOS_PORT
+ - VIDEOS_URL
+ depends_on:
+ - traefik
+ - redis
+ labels:
+ # Enable Traefik
+ - 'traefik.enable=true'
+ # Traefik routing for the videos service at /v1/videos
+ - 'traefik.http.routers.videos.rule=PathPrefix(`/${API_VERSION}/videos`)'
+ # Specify the posts service port
+ - 'traefik.http.services.videos.loadbalancer.server.port=${VIDEOS_PORT}'
+ # Add middleware to this route to strip the /v1/posts prefix
+ - 'traefik.http.middlewares.strip_videos_prefix.stripprefix.prefixes=/${API_VERSION}/videos'
+ - 'traefik.http.middlewares.strip_videos_prefix.stripprefix.forceSlash=true'
+ - 'traefik.http.routers.videos.middlewares=strip_videos_prefix'
+
##############################################################################
# Third-Party Dependencies and Support Services
##############################################################################
@@ -285,6 +310,8 @@ services:
- SEARCH_URL=${SEARCH_URL}
- FEED_DISCOVERY_URL=${FEED_DISCOVERY_URL}
- STATUS_URL=${STATUS_URL}
+ - VIDEOS_URL=${VIDEOS_URL}
+
container_name: 'nginx'
environment:
- TELESCOPE_HOST
diff --git a/src/api/videos/.dockerignore b/src/api/videos/.dockerignore
new file mode 100644
index 0000000000..b5d1637224
--- /dev/null
+++ b/src/api/videos/.dockerignore
@@ -0,0 +1,6 @@
+.dockerignore
+node_modules
+npm-debug.log
+Dockerfile
+.git
+.gitignore
diff --git a/src/api/videos/.gitignore b/src/api/videos/.gitignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/src/api/videos/Dockerfile b/src/api/videos/Dockerfile
new file mode 100644
index 0000000000..1a42450cc1
--- /dev/null
+++ b/src/api/videos/Dockerfile
@@ -0,0 +1,14 @@
+FROM node:lts-alpine as base
+
+# https://snyk.io/blog/10-best-practices-to-containerize-nodejs-web-applications-with-docker/
+RUN apk add dumb-init
+
+WORKDIR /app
+
+COPY --chown=node:node . .
+
+RUN npm install --only=production --no-package-lock
+
+USER node
+
+CMD ["dumb-init", "node", "src/server.js"]
diff --git a/src/api/videos/README.md b/src/api/videos/README.md
new file mode 100644
index 0000000000..d02af89b4f
--- /dev/null
+++ b/src/api/videos/README.md
@@ -0,0 +1,36 @@
+# Videos Service
+
+The Video Service returns YouTube links to videos uploaded from user's registered channels.
+Its main purpose is to embed a YouTube video player on Telescope.
+This service is really similar to the Post service.
+
+## Install
+
+```
+npm install
+```
+
+## Usage
+
+```
+# normal mode
+npm start
+
+# dev mode with automatic restarts
+npm run dev
+```
+
+By default, the server will run on .
+
+### Available routes with examples
+
+- `GET /videos` - Returns the 10 latests videos tracked. Query parameters:
+ - `page` - a number. It is a number from 1 or greater. It specifies the page you want to see
+ from the results.
+ - `per_page` - a number. It is a number from 1 or greater. It specifies how many videos should be returned by page.
+- `GET /videos/:id` - Returns the information associated to the video ID.
+
+## Docker
+
+- To build and tag: `docker build . -t telescope_videos_svc:latest`
+- To run locally: `docker run -p 11234:11234 telescope_videos_svc:latest`
diff --git a/src/api/videos/jest.config.e2e.js b/src/api/videos/jest.config.e2e.js
new file mode 100644
index 0000000000..28e2b1d81c
--- /dev/null
+++ b/src/api/videos/jest.config.e2e.js
@@ -0,0 +1,8 @@
+const baseConfig = require('../../../jest.config.base');
+
+module.exports = {
+ ...baseConfig,
+ rootDir: '../../..',
+ testMatch: ['/src/api/videos/test/e2e/*.test.js'],
+ collectCoverageFrom: ['/src/api/videos/src/**/*.js'],
+};
diff --git a/src/api/videos/jest.config.js b/src/api/videos/jest.config.js
new file mode 100644
index 0000000000..decd6cae2f
--- /dev/null
+++ b/src/api/videos/jest.config.js
@@ -0,0 +1,8 @@
+const baseConfig = require('../../../jest.config.base');
+
+module.exports = {
+ ...baseConfig,
+ rootDir: '../../..',
+ testMatch: ['/src/api/videos/test/*.test.js'],
+ collectCoverageFrom: ['/src/api/videos/src/**/*.js'],
+};
diff --git a/src/api/videos/package.json b/src/api/videos/package.json
new file mode 100644
index 0000000000..b61abe3b70
--- /dev/null
+++ b/src/api/videos/package.json
@@ -0,0 +1,29 @@
+{
+ "name": "@senecacdot/videos-service",
+ "version": "1.0.0",
+ "private": true,
+ "description": "A service for retrieving videos from YouTube",
+ "scripts": {
+ "start": "node src/server.js"
+ },
+ "repository": "Seneca-CDOT/telescope",
+ "license": "BSD-2-Clause",
+ "bugs": {
+ "url": "https://github.com/Seneca-CDOT/telescope/issues"
+ },
+ "homepage": "https://github.com/Seneca-CDOT/telescope#readme",
+ "engines": {
+ "node": ">=12.0.0"
+ },
+ "dependencies": {
+ "@senecacdot/satellite": "^1.x",
+ "express-validator": "^6.10.0",
+ "ioredis": "^4.25.0",
+ "ioredis-mock": "^5.4.1",
+ "jsdom": "^18.0.0",
+ "normalize-url": "^6.0.1"
+ },
+ "devDependencies": {
+ "supertest": "^6.1.3"
+ }
+}
diff --git a/src/api/videos/src/index.js b/src/api/videos/src/index.js
new file mode 100644
index 0000000000..f4afac7f05
--- /dev/null
+++ b/src/api/videos/src/index.js
@@ -0,0 +1,8 @@
+const { Satellite } = require('@senecacdot/satellite');
+
+const service = new Satellite();
+const videos = require('./routes/videos');
+
+service.router.use('/', videos);
+
+module.exports = service;
diff --git a/src/api/videos/src/models/feed.js b/src/api/videos/src/models/feed.js
new file mode 100644
index 0000000000..116c896217
--- /dev/null
+++ b/src/api/videos/src/models/feed.js
@@ -0,0 +1,43 @@
+const normalizeUrl = require('normalize-url');
+const { hash } = require('@senecacdot/satellite');
+
+const { getFeed } = require('../storage');
+
+const urlToId = (url) => hash(normalizeUrl(url));
+
+class Feed {
+ constructor(author, url, user, link, etag, lastModified) {
+ if (!url) {
+ throw new Error('missing url for feed');
+ }
+ if (!author) {
+ throw new Error('missing author for feed');
+ }
+ // Use the feed's normalized url as our unique identifier
+ this.id = urlToId(url);
+ this.author = author;
+ this.url = url;
+ this.user = user;
+ this.link = link;
+
+ // We may or may not have these cache values when we create a feed.
+ this.etag = etag === '' ? null : etag;
+ this.lastModified = lastModified === '' ? null : lastModified;
+ }
+
+ /**
+ * Returns a Feed from the database using the given id
+ * @param {String} id - the id of a feed (hashed, normalized url) to get from Redis.
+ * Returns a Promise
+ */
+ static async byId(id) {
+ const data = await getFeed(id);
+ // No feed found using this id
+ if (!(data && data.id)) {
+ return null;
+ }
+ return new Feed(data.author, data.url, data.user, data.link, data.etag, data.lastModified);
+ }
+}
+
+module.exports = Feed;
diff --git a/src/api/videos/src/models/video.js b/src/api/videos/src/models/video.js
new file mode 100644
index 0000000000..55a5f8199e
--- /dev/null
+++ b/src/api/videos/src/models/video.js
@@ -0,0 +1,65 @@
+const Feed = require('./feed');
+const { hash } = require('@senecacdot/satellite');
+const { getVideo } = require('../storage');
+
+/**
+ * Makes sure that a given date can be constructed as a Date object
+ * Returns a constructed Date object, if possible
+ * Otherwise throws an Error
+ * @param {Object} date an Object to construct as a Date object
+ * @param {Date} [fallbackDate] an optional second Date to construct in case the first fails to do so
+ */
+function ensureDate(date, fallbackDate) {
+ if (
+ date &&
+ (Object.prototype.toString.call(date) === '[object String]' ||
+ (Object.prototype.toString.call(date) === '[object Date]' && !Number.isNaN(date)))
+ ) {
+ return new Date(date);
+ }
+ if (Object.prototype.toString.call(fallbackDate) === '[object Date]') {
+ return new Date(fallbackDate);
+ }
+
+ throw new Error(`video has an invalid date: ${date}'`);
+}
+
+/**
+ * Makes sure that the given feed is a Feed and not just an id. If the latter
+ * it gets the full feed.
+ * @param {Feed|String} feed a Feed object or feed id
+ * Returns a Promise
+ */
+function ensureFeed(feed) {
+ return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed);
+}
+
+class Video {
+ constructor(title, url, guid, datePublished, dateUpdated, feed) {
+ this.id = hash(guid);
+ this.title = title;
+ this.url = url;
+ this.guid = guid;
+ this.published = ensureDate(datePublished);
+ this.updated = ensureDate(dateUpdated, datePublished);
+
+ if (!(feed instanceof Feed)) {
+ throw new Error(`expected feed to be a Feed Object, got '${feed}'`);
+ }
+ this.feed = feed;
+ }
+
+ static async byId(id) {
+ const data = await getVideo(id);
+
+ if (!(data && data.id)) {
+ return null;
+ }
+
+ const feed = await ensureFeed(data.feed);
+ const video = new Video(data.title, data.url, data.guid, data.published, data.updated, feed);
+ return video;
+ }
+}
+
+module.exports = Video;
diff --git a/src/api/videos/src/redis.js b/src/api/videos/src/redis.js
new file mode 100644
index 0000000000..7b7fb9c78f
--- /dev/null
+++ b/src/api/videos/src/redis.js
@@ -0,0 +1,5 @@
+const { Redis } = require('@senecacdot/satellite');
+
+const redis = Redis();
+
+module.exports = redis;
diff --git a/src/api/videos/src/routes/videos.js b/src/api/videos/src/routes/videos.js
new file mode 100644
index 0000000000..3ac588a570
--- /dev/null
+++ b/src/api/videos/src/routes/videos.js
@@ -0,0 +1,83 @@
+const { Router, createError, logger } = require('@senecacdot/satellite');
+const Video = require('../models/video');
+const { getVideos, getVideosCount } = require('../storage');
+const { validateVideosQuery, validateVideosIdParam } = require('../validation');
+
+const videosUrl = process.env.VIDEOS_URL || '/';
+
+const videos = Router();
+
+videos.get('/', validateVideosQuery(), async (req, res, next) => {
+ const defaultNumberOfVideos = process.env.MAX_VIDEOS_PER_PAGE || 30;
+ const capNumOfVideos = 100;
+ const page = parseInt(req.query.page || 1, 10);
+
+ let ids;
+ let perPage;
+ let videosCount = 0;
+ let from;
+ let to;
+
+ if (req.query.per_page) {
+ perPage = req.query.per_page > capNumOfVideos ? capNumOfVideos : req.query.per_page;
+ } else {
+ perPage = defaultNumberOfVideos;
+ }
+
+ try {
+ videosCount = await getVideosCount();
+
+ from = perPage * (page - 1);
+
+ to = perPage * page > videosCount ? videosCount : perPage * page;
+
+ ids = await getVideos(from, to);
+ } catch (err) {
+ logger.error({ err }, 'Unable to get videos from Redis');
+ next(createError(503, 'Unable to connect to database'));
+ }
+
+ const nextPage = to >= videosCount ? 1 : page + 1;
+ const prevPage = from === 0 ? Math.floor(videosCount / perPage) : page - 1;
+
+ res.set('X-Total-Count', videosCount);
+
+ res.links({
+ next: `/videos?per_page=${perPage}&page=${nextPage}`,
+ prev: `/videos?per_page=${perPage}&page=${prevPage}`,
+ first: `/videos?per_page=${perPage}&page=${1}`,
+ last: `/videos?per_page=${perPage}&page=${Math.floor(videosCount / perPage)}`,
+ });
+
+ res.json(ids.map((id) => ({ id, url: `${videosUrl}/${id}` })));
+});
+
+videos.get('/:id', validateVideosIdParam(), async (req, res, next) => {
+ const { id } = req.params;
+
+ try {
+ const video = await Video.byId(id);
+
+ if (!video) {
+ res.status(404).json({
+ message: `Video not found for id ${id}`,
+ });
+ } else {
+ switch (req.accepts(['json'])) {
+ case 'json':
+ res.append('Content-type', 'application/json').json(video);
+ break;
+ default:
+ res.status(406).json({
+ message: 'Invalid content type',
+ });
+ break;
+ }
+ }
+ } catch (err) {
+ logger.error({ err }, 'Unable to get videos from Redis');
+ next(createError(503, 'Unable to connect to database'));
+ }
+});
+
+module.exports = videos;
diff --git a/src/api/videos/src/server.js b/src/api/videos/src/server.js
new file mode 100644
index 0000000000..73a2cd60d4
--- /dev/null
+++ b/src/api/videos/src/server.js
@@ -0,0 +1,10 @@
+const { logger } = require('@senecacdot/satellite');
+const service = require('.');
+
+const PORT = parseInt(process.env.VIDEOS_PORT || 11234, 10);
+
+service.start(PORT, () => {
+ logger.debug(`Videos microservice started on port ${PORT}`);
+});
+
+module.exports = service;
diff --git a/src/api/videos/src/storage.js b/src/api/videos/src/storage.js
new file mode 100644
index 0000000000..f22e1f0446
--- /dev/null
+++ b/src/api/videos/src/storage.js
@@ -0,0 +1,228 @@
+const { logger } = require('@senecacdot/satellite');
+const redis = require('./redis');
+// Redis Keys
+const feedsKey = 't:feeds';
+const flaggedFeedsKey = 't:feeds:flagged';
+const postsKey = 't:posts';
+const videosKey = 't:videos';
+
+// Namespaces
+const feedNamespace = 't:feed:';
+const postNamespace = 't:post:';
+const videoNamespace = 't:video:';
+// Suffixes
+const invalidSuffix = ':invalid';
+const delayedSuffix = ':delayed';
+
+// "6Xoj0UXOW3" to "t:post:6Xoj0UXOW3"
+const createPostKey = (id) => postNamespace.concat(id);
+// "MpoY0ZxK23" to "t:video:MpoY0ZxK23"
+const createVideoKey = (id) => videoNamespace.concat(id);
+// "NirlSYranl" to "t:feed:NirlSYranl"
+const createFeedKey = (id) => feedNamespace.concat(id);
+// "NirlSYranl" to "t:feed:NirlSYranl:invalid"
+const createInvalidFeedKey = (id) => createFeedKey(id).concat(invalidSuffix);
+// "NirlSYranl" to "t:feed:NirlSYranl:delayed"
+const createDelayedFeedKey = (id) => createFeedKey(id).concat(delayedSuffix);
+
+module.exports = {
+ /**
+ * Feeds
+ */
+ addFeed: async (feed) => {
+ // Check if feed being added already exists in flagged feeds set
+ // If it is, do nothing
+ if (await redis.sismember(flaggedFeedsKey, feed.id)) return;
+
+ const key = createFeedKey(feed.id);
+ await redis
+ .multi()
+ .hset(
+ key,
+ 'id',
+ feed.id,
+ 'author',
+ feed.author,
+ 'url',
+ feed.url,
+ 'user',
+ feed.user,
+ 'link',
+ feed.link,
+ 'etag',
+ feed.etag,
+ 'lastModified',
+ feed.lastModified
+ )
+ .sadd(feedsKey, feed.id)
+ .exec();
+ },
+
+ getFeeds: () => redis.smembers(feedsKey),
+
+ getFlaggedFeeds: () => redis.smembers(flaggedFeedsKey),
+
+ getFeed: (id) => redis.hgetall(feedNamespace.concat(id)),
+
+ getFeedsCount: () => redis.scard(feedsKey),
+
+ setInvalidFeed: (id, reason) => {
+ const key = createInvalidFeedKey(id);
+ return redis.set(key, reason);
+ },
+
+ /**
+ * Removes a feed entry from redis
+ * @param id id of feed to be removed
+ */
+ removeFeed: async (id) => {
+ const key = createFeedKey(id);
+ // Checks which set the feed is currently in
+ const redisKey = (await redis.sismember(feedsKey, id)) ? feedsKey : flaggedFeedsKey;
+ try {
+ await redis
+ .multi()
+ .hdel(key, 'id', 'author', 'url', 'user', 'link', 'etag', 'lastModified')
+ .srem(redisKey, id)
+ .exec();
+ } catch (error) {
+ logger.error({ error }, `Error removing Feed ${id} from Redis`);
+ throw new Error(`Error trying to remove feed from Redis`);
+ }
+ },
+
+ setFlaggedFeed: (id) => redis.smove(feedsKey, flaggedFeedsKey, id),
+
+ unsetFlaggedFeed: (id) => redis.smove(flaggedFeedsKey, feedsKey, id),
+
+ isInvalid: (id) => redis.exists(createInvalidFeedKey(id)),
+
+ setDelayedFeed: (id, seconds) => redis.set(createDelayedFeedKey(id), seconds, 1),
+
+ isDelayed: (id) => redis.exists(createDelayedFeedKey(id)),
+
+ /**
+ * Posts
+ */
+ addPost: async (post) => {
+ const key = createPostKey(post.id);
+ await redis
+ .multi()
+ .hset(
+ key,
+ 'id',
+ post.id,
+ 'title',
+ post.title,
+ 'html',
+ post.html,
+ 'published',
+ post.published,
+ 'updated',
+ post.updated,
+ 'url',
+ post.url,
+ 'guid',
+ post.guid,
+ 'feed',
+ post.feed
+ )
+ // sort set by published date as scores
+ .zadd(postsKey, post.published.getTime(), post.id)
+ .exec();
+ },
+
+ /**
+ * Returns an array of post ids from redis
+ * @param from lower index
+ * @param to higher index, it needs -1 because redis includes the element at this index in the returned array
+ * @return Array of ids
+ */
+ getPosts: (from, to) => redis.zrevrange(postsKey, from, to - 1),
+
+ /**
+ * Returns an array of post ids from redis
+ * @param start starting Date in the range
+ * @param end ending Date in the range
+ * @return Array of ids
+ */
+ getPostsByDate: (startDate, endDate) =>
+ redis.zrangebyscore(postsKey, startDate.getTime(), endDate.getTime()),
+
+ getPostsCount: () => redis.zcard(postsKey),
+
+ getPost: (id) => redis.hgetall(postNamespace.concat(id)),
+
+ /**
+ * Removes a post entry from redis
+ * NOTE: removing a post from redis should also require the post to be removed from ElasticSearch
+ * @param id id of post to be removed
+ */
+ removePost: async (id) => {
+ const key = createPostKey(id);
+ await redis
+ .multi()
+ .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed')
+ .zrem(postsKey, id)
+ .exec();
+ },
+
+ /**
+ * Introduces a new Video record
+ * to the 't:video' namespace.
+ *
+ * The structure of video is of a Video-like object.
+ *
+ * @param {*} video a Video-like object
+ */
+ addVideo: async (video) => {
+ const key = createVideoKey(video.id);
+ await redis
+ .multi()
+ .hset(
+ key,
+ 'id',
+ video.id,
+ 'title',
+ video.title,
+ 'url',
+ video.url,
+ 'published',
+ video.published,
+ 'updated',
+ video.updated,
+ 'guid',
+ video.guid,
+ 'feed',
+ video.feed
+ )
+ // sort set by published date as scores
+ .zadd(videosKey, video.published.getTime(), video.id)
+ .exec();
+ },
+
+ /**
+ * Returns an array of video ids, where the range is 'from' index
+ * to 'to' index, exclusive; that is, 'from ..< to'
+ *
+ * @param {*} from starting index, included in the result
+ * @param {*} to ending index, excluded from the result
+ * @returns an array of video ids, length equal to "to - from"
+ */
+ getVideos: (from, to) => redis.zrevrange(videosKey, from, to - 1),
+
+ /**
+ * Returns an array of video ids, where each video has been uploaded
+ * between the 'startDate' and the 'endDate'
+ *
+ * @param {*} startDate
+ * @param {*} endDate
+ * @returns an array of video ids, length can vary
+ */
+ getVideosByDate: (startDate, endDate) =>
+ redis.zrangebyscore(videosKey, startDate.getTime(), endDate.getTime()),
+
+ getVideosCount: () => redis.zcard(videosKey),
+
+ getVideo: (id) => redis.hgetall(videoNamespace.concat(id)),
+};
diff --git a/src/api/videos/src/validation.js b/src/api/videos/src/validation.js
new file mode 100644
index 0000000000..95dd199be2
--- /dev/null
+++ b/src/api/videos/src/validation.js
@@ -0,0 +1,33 @@
+const { param, validationResult, query } = require('express-validator');
+
+const validate = (rules) => {
+ return async (req, res, next) => {
+ await Promise.all(rules.map((rule) => rule.run(req)));
+
+ const result = validationResult(req);
+ if (result.isEmpty()) {
+ return next();
+ }
+
+ const errors = result.array();
+ return res.status(400).send(errors);
+ };
+};
+
+const videosQueryValidationRules = [
+ query('per_page', 'per_page needs to be empty or a integer').custom(
+ (value) => !value || Number.isInteger(+value)
+ ),
+ query('page', 'page needs to be empty or an integer').custom(
+ (value) => !value || Number.isInteger(+value)
+ ),
+];
+
+const videosIdParamValidationRules = [
+ param('id', 'Id Length is invalid').isLength({ min: 10, max: 10 }),
+];
+
+module.exports = {
+ validateVideosQuery: () => validate(videosQueryValidationRules),
+ validateVideosIdParam: () => validate(videosIdParamValidationRules),
+};