diff --git a/config/env.development b/config/env.development index 137caf5435..f6f514a423 100644 --- a/config/env.development +++ b/config/env.development @@ -155,6 +155,15 @@ PARSER_PORT=10000 # Parser Service URL PARSER_URL=http://localhost/v1/parser +################################################################################ +# Videos Service +################################################################################ + +# Video Service Port (default is 11234) +VIDEOS_PORT=11234 + +# Video Service URL +VIDEOS_URL=http://localhost/v1/videos ################################################################################ # Planet Service diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c9409e6a43..d9b53f0e57 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -263,6 +263,31 @@ services: - PLANET_PORT - POSTS_URL + # videos service + videos: + container_name: 'videos' + build: + context: ../src/api/videos + dockerfile: Dockerfile + environment: + - NODE_ENV + - VIDEOS_PORT + - VIDEOS_URL + depends_on: + - traefik + - redis + labels: + # Enable Traefik + - 'traefik.enable=true' + # Traefik routing for the videos service at /v1/videos + - 'traefik.http.routers.videos.rule=PathPrefix(`/${API_VERSION}/videos`)' + # Specify the posts service port + - 'traefik.http.services.videos.loadbalancer.server.port=${VIDEOS_PORT}' + # Add middleware to this route to strip the /v1/posts prefix + - 'traefik.http.middlewares.strip_videos_prefix.stripprefix.prefixes=/${API_VERSION}/videos' + - 'traefik.http.middlewares.strip_videos_prefix.stripprefix.forceSlash=true' + - 'traefik.http.routers.videos.middlewares=strip_videos_prefix' + ############################################################################## # Third-Party Dependencies and Support Services ############################################################################## @@ -285,6 +310,8 @@ services: - SEARCH_URL=${SEARCH_URL} - FEED_DISCOVERY_URL=${FEED_DISCOVERY_URL} - STATUS_URL=${STATUS_URL} + - VIDEOS_URL=${VIDEOS_URL} + container_name: 'nginx' environment: - TELESCOPE_HOST diff --git a/src/api/videos/.dockerignore b/src/api/videos/.dockerignore new file mode 100644 index 0000000000..b5d1637224 --- /dev/null +++ b/src/api/videos/.dockerignore @@ -0,0 +1,6 @@ +.dockerignore +node_modules +npm-debug.log +Dockerfile +.git +.gitignore diff --git a/src/api/videos/.gitignore b/src/api/videos/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/api/videos/Dockerfile b/src/api/videos/Dockerfile new file mode 100644 index 0000000000..1a42450cc1 --- /dev/null +++ b/src/api/videos/Dockerfile @@ -0,0 +1,14 @@ +FROM node:lts-alpine as base + +# https://snyk.io/blog/10-best-practices-to-containerize-nodejs-web-applications-with-docker/ +RUN apk add dumb-init + +WORKDIR /app + +COPY --chown=node:node . . + +RUN npm install --only=production --no-package-lock + +USER node + +CMD ["dumb-init", "node", "src/server.js"] diff --git a/src/api/videos/README.md b/src/api/videos/README.md new file mode 100644 index 0000000000..d02af89b4f --- /dev/null +++ b/src/api/videos/README.md @@ -0,0 +1,36 @@ +# Videos Service + +The Video Service returns YouTube links to videos uploaded from user's registered channels. +Its main purpose is to embed a YouTube video player on Telescope. +This service is really similar to the Post service. + +## Install + +``` +npm install +``` + +## Usage + +``` +# normal mode +npm start + +# dev mode with automatic restarts +npm run dev +``` + +By default, the server will run on . + +### Available routes with examples + +- `GET /videos` - Returns the 10 latests videos tracked. Query parameters: + - `page` - a number. It is a number from 1 or greater. It specifies the page you want to see + from the results. + - `per_page` - a number. It is a number from 1 or greater. It specifies how many videos should be returned by page. +- `GET /videos/:id` - Returns the information associated to the video ID. + +## Docker + +- To build and tag: `docker build . -t telescope_videos_svc:latest` +- To run locally: `docker run -p 11234:11234 telescope_videos_svc:latest` diff --git a/src/api/videos/jest.config.e2e.js b/src/api/videos/jest.config.e2e.js new file mode 100644 index 0000000000..28e2b1d81c --- /dev/null +++ b/src/api/videos/jest.config.e2e.js @@ -0,0 +1,8 @@ +const baseConfig = require('../../../jest.config.base'); + +module.exports = { + ...baseConfig, + rootDir: '../../..', + testMatch: ['/src/api/videos/test/e2e/*.test.js'], + collectCoverageFrom: ['/src/api/videos/src/**/*.js'], +}; diff --git a/src/api/videos/jest.config.js b/src/api/videos/jest.config.js new file mode 100644 index 0000000000..decd6cae2f --- /dev/null +++ b/src/api/videos/jest.config.js @@ -0,0 +1,8 @@ +const baseConfig = require('../../../jest.config.base'); + +module.exports = { + ...baseConfig, + rootDir: '../../..', + testMatch: ['/src/api/videos/test/*.test.js'], + collectCoverageFrom: ['/src/api/videos/src/**/*.js'], +}; diff --git a/src/api/videos/package.json b/src/api/videos/package.json new file mode 100644 index 0000000000..b61abe3b70 --- /dev/null +++ b/src/api/videos/package.json @@ -0,0 +1,29 @@ +{ + "name": "@senecacdot/videos-service", + "version": "1.0.0", + "private": true, + "description": "A service for retrieving videos from YouTube", + "scripts": { + "start": "node src/server.js" + }, + "repository": "Seneca-CDOT/telescope", + "license": "BSD-2-Clause", + "bugs": { + "url": "https://github.com/Seneca-CDOT/telescope/issues" + }, + "homepage": "https://github.com/Seneca-CDOT/telescope#readme", + "engines": { + "node": ">=12.0.0" + }, + "dependencies": { + "@senecacdot/satellite": "^1.x", + "express-validator": "^6.10.0", + "ioredis": "^4.25.0", + "ioredis-mock": "^5.4.1", + "jsdom": "^18.0.0", + "normalize-url": "^6.0.1" + }, + "devDependencies": { + "supertest": "^6.1.3" + } +} diff --git a/src/api/videos/src/index.js b/src/api/videos/src/index.js new file mode 100644 index 0000000000..f4afac7f05 --- /dev/null +++ b/src/api/videos/src/index.js @@ -0,0 +1,8 @@ +const { Satellite } = require('@senecacdot/satellite'); + +const service = new Satellite(); +const videos = require('./routes/videos'); + +service.router.use('/', videos); + +module.exports = service; diff --git a/src/api/videos/src/models/feed.js b/src/api/videos/src/models/feed.js new file mode 100644 index 0000000000..116c896217 --- /dev/null +++ b/src/api/videos/src/models/feed.js @@ -0,0 +1,43 @@ +const normalizeUrl = require('normalize-url'); +const { hash } = require('@senecacdot/satellite'); + +const { getFeed } = require('../storage'); + +const urlToId = (url) => hash(normalizeUrl(url)); + +class Feed { + constructor(author, url, user, link, etag, lastModified) { + if (!url) { + throw new Error('missing url for feed'); + } + if (!author) { + throw new Error('missing author for feed'); + } + // Use the feed's normalized url as our unique identifier + this.id = urlToId(url); + this.author = author; + this.url = url; + this.user = user; + this.link = link; + + // We may or may not have these cache values when we create a feed. + this.etag = etag === '' ? null : etag; + this.lastModified = lastModified === '' ? null : lastModified; + } + + /** + * Returns a Feed from the database using the given id + * @param {String} id - the id of a feed (hashed, normalized url) to get from Redis. + * Returns a Promise + */ + static async byId(id) { + const data = await getFeed(id); + // No feed found using this id + if (!(data && data.id)) { + return null; + } + return new Feed(data.author, data.url, data.user, data.link, data.etag, data.lastModified); + } +} + +module.exports = Feed; diff --git a/src/api/videos/src/models/video.js b/src/api/videos/src/models/video.js new file mode 100644 index 0000000000..55a5f8199e --- /dev/null +++ b/src/api/videos/src/models/video.js @@ -0,0 +1,65 @@ +const Feed = require('./feed'); +const { hash } = require('@senecacdot/satellite'); +const { getVideo } = require('../storage'); + +/** + * Makes sure that a given date can be constructed as a Date object + * Returns a constructed Date object, if possible + * Otherwise throws an Error + * @param {Object} date an Object to construct as a Date object + * @param {Date} [fallbackDate] an optional second Date to construct in case the first fails to do so + */ +function ensureDate(date, fallbackDate) { + if ( + date && + (Object.prototype.toString.call(date) === '[object String]' || + (Object.prototype.toString.call(date) === '[object Date]' && !Number.isNaN(date))) + ) { + return new Date(date); + } + if (Object.prototype.toString.call(fallbackDate) === '[object Date]') { + return new Date(fallbackDate); + } + + throw new Error(`video has an invalid date: ${date}'`); +} + +/** + * Makes sure that the given feed is a Feed and not just an id. If the latter + * it gets the full feed. + * @param {Feed|String} feed a Feed object or feed id + * Returns a Promise + */ +function ensureFeed(feed) { + return feed instanceof Feed ? Promise.resolve(feed) : Feed.byId(feed); +} + +class Video { + constructor(title, url, guid, datePublished, dateUpdated, feed) { + this.id = hash(guid); + this.title = title; + this.url = url; + this.guid = guid; + this.published = ensureDate(datePublished); + this.updated = ensureDate(dateUpdated, datePublished); + + if (!(feed instanceof Feed)) { + throw new Error(`expected feed to be a Feed Object, got '${feed}'`); + } + this.feed = feed; + } + + static async byId(id) { + const data = await getVideo(id); + + if (!(data && data.id)) { + return null; + } + + const feed = await ensureFeed(data.feed); + const video = new Video(data.title, data.url, data.guid, data.published, data.updated, feed); + return video; + } +} + +module.exports = Video; diff --git a/src/api/videos/src/redis.js b/src/api/videos/src/redis.js new file mode 100644 index 0000000000..7b7fb9c78f --- /dev/null +++ b/src/api/videos/src/redis.js @@ -0,0 +1,5 @@ +const { Redis } = require('@senecacdot/satellite'); + +const redis = Redis(); + +module.exports = redis; diff --git a/src/api/videos/src/routes/videos.js b/src/api/videos/src/routes/videos.js new file mode 100644 index 0000000000..3ac588a570 --- /dev/null +++ b/src/api/videos/src/routes/videos.js @@ -0,0 +1,83 @@ +const { Router, createError, logger } = require('@senecacdot/satellite'); +const Video = require('../models/video'); +const { getVideos, getVideosCount } = require('../storage'); +const { validateVideosQuery, validateVideosIdParam } = require('../validation'); + +const videosUrl = process.env.VIDEOS_URL || '/'; + +const videos = Router(); + +videos.get('/', validateVideosQuery(), async (req, res, next) => { + const defaultNumberOfVideos = process.env.MAX_VIDEOS_PER_PAGE || 30; + const capNumOfVideos = 100; + const page = parseInt(req.query.page || 1, 10); + + let ids; + let perPage; + let videosCount = 0; + let from; + let to; + + if (req.query.per_page) { + perPage = req.query.per_page > capNumOfVideos ? capNumOfVideos : req.query.per_page; + } else { + perPage = defaultNumberOfVideos; + } + + try { + videosCount = await getVideosCount(); + + from = perPage * (page - 1); + + to = perPage * page > videosCount ? videosCount : perPage * page; + + ids = await getVideos(from, to); + } catch (err) { + logger.error({ err }, 'Unable to get videos from Redis'); + next(createError(503, 'Unable to connect to database')); + } + + const nextPage = to >= videosCount ? 1 : page + 1; + const prevPage = from === 0 ? Math.floor(videosCount / perPage) : page - 1; + + res.set('X-Total-Count', videosCount); + + res.links({ + next: `/videos?per_page=${perPage}&page=${nextPage}`, + prev: `/videos?per_page=${perPage}&page=${prevPage}`, + first: `/videos?per_page=${perPage}&page=${1}`, + last: `/videos?per_page=${perPage}&page=${Math.floor(videosCount / perPage)}`, + }); + + res.json(ids.map((id) => ({ id, url: `${videosUrl}/${id}` }))); +}); + +videos.get('/:id', validateVideosIdParam(), async (req, res, next) => { + const { id } = req.params; + + try { + const video = await Video.byId(id); + + if (!video) { + res.status(404).json({ + message: `Video not found for id ${id}`, + }); + } else { + switch (req.accepts(['json'])) { + case 'json': + res.append('Content-type', 'application/json').json(video); + break; + default: + res.status(406).json({ + message: 'Invalid content type', + }); + break; + } + } + } catch (err) { + logger.error({ err }, 'Unable to get videos from Redis'); + next(createError(503, 'Unable to connect to database')); + } +}); + +module.exports = videos; diff --git a/src/api/videos/src/server.js b/src/api/videos/src/server.js new file mode 100644 index 0000000000..73a2cd60d4 --- /dev/null +++ b/src/api/videos/src/server.js @@ -0,0 +1,10 @@ +const { logger } = require('@senecacdot/satellite'); +const service = require('.'); + +const PORT = parseInt(process.env.VIDEOS_PORT || 11234, 10); + +service.start(PORT, () => { + logger.debug(`Videos microservice started on port ${PORT}`); +}); + +module.exports = service; diff --git a/src/api/videos/src/storage.js b/src/api/videos/src/storage.js new file mode 100644 index 0000000000..f22e1f0446 --- /dev/null +++ b/src/api/videos/src/storage.js @@ -0,0 +1,228 @@ +const { logger } = require('@senecacdot/satellite'); +const redis = require('./redis'); +// Redis Keys +const feedsKey = 't:feeds'; +const flaggedFeedsKey = 't:feeds:flagged'; +const postsKey = 't:posts'; +const videosKey = 't:videos'; + +// Namespaces +const feedNamespace = 't:feed:'; +const postNamespace = 't:post:'; +const videoNamespace = 't:video:'; +// Suffixes +const invalidSuffix = ':invalid'; +const delayedSuffix = ':delayed'; + +// "6Xoj0UXOW3" to "t:post:6Xoj0UXOW3" +const createPostKey = (id) => postNamespace.concat(id); +// "MpoY0ZxK23" to "t:video:MpoY0ZxK23" +const createVideoKey = (id) => videoNamespace.concat(id); +// "NirlSYranl" to "t:feed:NirlSYranl" +const createFeedKey = (id) => feedNamespace.concat(id); +// "NirlSYranl" to "t:feed:NirlSYranl:invalid" +const createInvalidFeedKey = (id) => createFeedKey(id).concat(invalidSuffix); +// "NirlSYranl" to "t:feed:NirlSYranl:delayed" +const createDelayedFeedKey = (id) => createFeedKey(id).concat(delayedSuffix); + +module.exports = { + /** + * Feeds + */ + addFeed: async (feed) => { + // Check if feed being added already exists in flagged feeds set + // If it is, do nothing + if (await redis.sismember(flaggedFeedsKey, feed.id)) return; + + const key = createFeedKey(feed.id); + await redis + .multi() + .hset( + key, + 'id', + feed.id, + 'author', + feed.author, + 'url', + feed.url, + 'user', + feed.user, + 'link', + feed.link, + 'etag', + feed.etag, + 'lastModified', + feed.lastModified + ) + .sadd(feedsKey, feed.id) + .exec(); + }, + + getFeeds: () => redis.smembers(feedsKey), + + getFlaggedFeeds: () => redis.smembers(flaggedFeedsKey), + + getFeed: (id) => redis.hgetall(feedNamespace.concat(id)), + + getFeedsCount: () => redis.scard(feedsKey), + + setInvalidFeed: (id, reason) => { + const key = createInvalidFeedKey(id); + return redis.set(key, reason); + }, + + /** + * Removes a feed entry from redis + * @param id id of feed to be removed + */ + removeFeed: async (id) => { + const key = createFeedKey(id); + // Checks which set the feed is currently in + const redisKey = (await redis.sismember(feedsKey, id)) ? feedsKey : flaggedFeedsKey; + try { + await redis + .multi() + .hdel(key, 'id', 'author', 'url', 'user', 'link', 'etag', 'lastModified') + .srem(redisKey, id) + .exec(); + } catch (error) { + logger.error({ error }, `Error removing Feed ${id} from Redis`); + throw new Error(`Error trying to remove feed from Redis`); + } + }, + + setFlaggedFeed: (id) => redis.smove(feedsKey, flaggedFeedsKey, id), + + unsetFlaggedFeed: (id) => redis.smove(flaggedFeedsKey, feedsKey, id), + + isInvalid: (id) => redis.exists(createInvalidFeedKey(id)), + + setDelayedFeed: (id, seconds) => redis.set(createDelayedFeedKey(id), seconds, 1), + + isDelayed: (id) => redis.exists(createDelayedFeedKey(id)), + + /** + * Posts + */ + addPost: async (post) => { + const key = createPostKey(post.id); + await redis + .multi() + .hset( + key, + 'id', + post.id, + 'title', + post.title, + 'html', + post.html, + 'published', + post.published, + 'updated', + post.updated, + 'url', + post.url, + 'guid', + post.guid, + 'feed', + post.feed + ) + // sort set by published date as scores + .zadd(postsKey, post.published.getTime(), post.id) + .exec(); + }, + + /** + * Returns an array of post ids from redis + * @param from lower index + * @param to higher index, it needs -1 because redis includes the element at this index in the returned array + * @return Array of ids + */ + getPosts: (from, to) => redis.zrevrange(postsKey, from, to - 1), + + /** + * Returns an array of post ids from redis + * @param start starting Date in the range + * @param end ending Date in the range + * @return Array of ids + */ + getPostsByDate: (startDate, endDate) => + redis.zrangebyscore(postsKey, startDate.getTime(), endDate.getTime()), + + getPostsCount: () => redis.zcard(postsKey), + + getPost: (id) => redis.hgetall(postNamespace.concat(id)), + + /** + * Removes a post entry from redis + * NOTE: removing a post from redis should also require the post to be removed from ElasticSearch + * @param id id of post to be removed + */ + removePost: async (id) => { + const key = createPostKey(id); + await redis + .multi() + .hdel(key, 'id', 'title', 'html', 'published', 'updated', 'url', 'guid', 'feed') + .zrem(postsKey, id) + .exec(); + }, + + /** + * Introduces a new Video record + * to the 't:video' namespace. + * + * The structure of video is of a Video-like object. + * + * @param {*} video a Video-like object + */ + addVideo: async (video) => { + const key = createVideoKey(video.id); + await redis + .multi() + .hset( + key, + 'id', + video.id, + 'title', + video.title, + 'url', + video.url, + 'published', + video.published, + 'updated', + video.updated, + 'guid', + video.guid, + 'feed', + video.feed + ) + // sort set by published date as scores + .zadd(videosKey, video.published.getTime(), video.id) + .exec(); + }, + + /** + * Returns an array of video ids, where the range is 'from' index + * to 'to' index, exclusive; that is, 'from ..< to' + * + * @param {*} from starting index, included in the result + * @param {*} to ending index, excluded from the result + * @returns an array of video ids, length equal to "to - from" + */ + getVideos: (from, to) => redis.zrevrange(videosKey, from, to - 1), + + /** + * Returns an array of video ids, where each video has been uploaded + * between the 'startDate' and the 'endDate' + * + * @param {*} startDate + * @param {*} endDate + * @returns an array of video ids, length can vary + */ + getVideosByDate: (startDate, endDate) => + redis.zrangebyscore(videosKey, startDate.getTime(), endDate.getTime()), + + getVideosCount: () => redis.zcard(videosKey), + + getVideo: (id) => redis.hgetall(videoNamespace.concat(id)), +}; diff --git a/src/api/videos/src/validation.js b/src/api/videos/src/validation.js new file mode 100644 index 0000000000..95dd199be2 --- /dev/null +++ b/src/api/videos/src/validation.js @@ -0,0 +1,33 @@ +const { param, validationResult, query } = require('express-validator'); + +const validate = (rules) => { + return async (req, res, next) => { + await Promise.all(rules.map((rule) => rule.run(req))); + + const result = validationResult(req); + if (result.isEmpty()) { + return next(); + } + + const errors = result.array(); + return res.status(400).send(errors); + }; +}; + +const videosQueryValidationRules = [ + query('per_page', 'per_page needs to be empty or a integer').custom( + (value) => !value || Number.isInteger(+value) + ), + query('page', 'page needs to be empty or an integer').custom( + (value) => !value || Number.isInteger(+value) + ), +]; + +const videosIdParamValidationRules = [ + param('id', 'Id Length is invalid').isLength({ min: 10, max: 10 }), +]; + +module.exports = { + validateVideosQuery: () => validate(videosQueryValidationRules), + validateVideosIdParam: () => validate(videosIdParamValidationRules), +};