Skip to content

Commit

Permalink
Migrated most of /scripts/lib to TypeScript
Browse files Browse the repository at this point in the history
  • Loading branch information
jwngr committed Jul 13, 2024
1 parent f56291b commit 1e07cb7
Show file tree
Hide file tree
Showing 46 changed files with 1,097 additions and 994 deletions.
12 changes: 7 additions & 5 deletions scripts/archive/fetchUniqueLocations.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import {transformForAllSeasons} from '../../website/src/resources/schedules';
import {Logger} from '../lib/logger';

Logger.info('Fetching unique locations...');
const logger = new Logger({isSentryEnabled: false});

logger.info('Fetching unique locations...');

let gamesCount = 0;
let locations = new Set();
Expand All @@ -16,12 +18,12 @@ transformForAllSeasons((gameData) => {
locations.add(`${city}|||${state || country}|||${stadium || ''}`);
});

Logger.info('GAMES COUNT:', gamesCount);
Logger.info('LOCATIONS COUNT:', locations.size);
logger.info('GAMES COUNT:', gamesCount);
logger.info('LOCATIONS COUNT:', locations.size);

locations.forEach((location) => {
const [city, stateOrCountry, stadium] = location.split('|||');
Logger.info(`${city}\t${stateOrCountry}\t${stadium}`);
logger.info(`${city}\t${stateOrCountry}\t${stadium}`);
});

Logger.success('Unique locations fetched!');
logger.success('Unique locations fetched!');
6 changes: 4 additions & 2 deletions scripts/archive/generateDecadeCsvs.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import {getForSeason} from '../../website/src/resources/schedules';
import {CURRENT_SEASON} from '../lib/constants';
import {Logger} from '../lib/logger';

const logger = new Logger({isSentryEnabled: false});

const OUTPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../data/decadeCsvs');

const DECADES = [
Expand Down Expand Up @@ -48,7 +50,7 @@ const stats = [
{name: 'possession', text: 'Possession'},
];

Logger.info('Generating CSVs...');
logger.info('Generating CSVs...');

DECADES.forEach((seasons) => {
const firstYear = seasons[0];
Expand Down Expand Up @@ -90,4 +92,4 @@ DECADES.forEach((seasons) => {
fs.writeFileSync(outputFilename, lines.join('\n'));
});

Logger.success('CSVs generated!');
logger.success('CSVs generated!');
14 changes: 8 additions & 6 deletions scripts/archive/scrapeGameDetails_WP.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import request from 'request-promise';

import {Logger} from '../lib/logger';

const logger = new Logger({isSentryEnabled: false});

const INPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../data/schedules');

const getHtmlForUrl = (url) => {
Expand All @@ -19,7 +21,7 @@ const getHtmlForUrl = (url) => {
};

const fetchGameDetailsForYear = (year) => {
Logger.info(`Fetching year ${year}.`);
logger.info(`Fetching year ${year}.`);

return getHtmlForUrl(
`https://en.wikipedia.org/wiki/${year}_Notre_Dame_Fighting_Irish_football_team`
Expand Down Expand Up @@ -99,7 +101,7 @@ const fetchGameDetailsForYear = (year) => {
const opponent = rowCellValues[opponentIndex];
const isHomeGame = !_.startsWith(opponent, 'at');
if (gamesData[i - 1].isHomeGame !== isHomeGame) {
// Logger.info('HOME / AWAY MISMATCH:', year, i - 1, opponent);
// logger.info('HOME / AWAY MISMATCH:', year, i - 1, opponent);
}
}

Expand All @@ -108,7 +110,7 @@ const fetchGameDetailsForYear = (year) => {
if (resultIndex !== -1) {
const result = rowCellValues[resultIndex][0];
if (gamesData[i - 1].result !== result) {
// Logger.info('RESULT MISMATCH:', year, i - 1);
// logger.info('RESULT MISMATCH:', year, i - 1);
}
}

Expand Down Expand Up @@ -152,22 +154,22 @@ const fetchGameDetailsForYear = (year) => {
}
}
} catch (error) {
Logger.error(`Failed to parse schedule for ${year}:`, {error, rowCellValues});
logger.error(`Failed to parse schedule for ${year}:`, {error, rowCellValues});
throw error;
}
}
});

fs.writeFileSync(filename, JSON.stringify(gamesData, null, 2));

Logger.info(`Success ${year}!`);
logger.info(`Success ${year}!`);
})
.catch((error) => {
let errorMessage = error.message;
if (error.statusCode === 404) {
errorMessage = '404 page not found.';
}
Logger.error(`Failed to fetch schedule for ${year}:`, {errorMessage});
logger.error(`Failed to fetch schedule for ${year}:`, {errorMessage});
});
};

Expand Down
8 changes: 5 additions & 3 deletions scripts/archive/scrapeGameIdsAndApRankings_SR.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import request from 'request-promise';
import {CURRENT_SEASON} from '../lib/constants';
import {Logger} from '../lib/logger';

const logger = new Logger({isSentryEnabled: false});

const INPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../website/src/resources/schedules');

const SPORTS_REFERENCE_GAME_STATS_START_YEAR = 2000;
Expand Down Expand Up @@ -58,7 +60,7 @@ const promises = years.map((year) => {
};
})
.catch((error) => {
Logger.error(`Error fetching game IDs and AP rankings for ${year}`, {error});
logger.error(`Error fetching game IDs and AP rankings for ${year}`, {error});
});
});

Expand Down Expand Up @@ -105,8 +107,8 @@ Promise.all(promises)
fs.writeFileSync(filename, JSON.stringify(yearData, null, 2));
});

Logger.success('Success!');
logger.success('Success!');
})
.catch((error) => {
Logger.fail(`Error fetching all game IDs`, error);
logger.fail(`Error fetching all game IDs`, error);
});
10 changes: 6 additions & 4 deletions scripts/archive/scrapeGameStats_SR.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import puppeteer from 'puppeteer';
import {CURRENT_SEASON} from '../lib/constants';
import {Logger} from '../lib/logger';

const logger = new Logger({isSentryEnabled: false});

const INPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../website/src/resources/schedules');

process.setMaxListeners(Infinity);
Expand All @@ -18,7 +20,7 @@ const scrapeGameStats = async (gameId) => {

const url = `https://www.sports-reference.com/cfb/boxscores/${gameId}.html`;

Logger.info(`Scraping ${url}`);
logger.info(`Scraping ${url}`);

await page.goto(url, {
waitUntil: 'networkidle2',
Expand Down Expand Up @@ -77,7 +79,7 @@ const fn = async () => {
const promises = _.map(yearData, (gameData) => {
if ('sportsReferenceGameId' in gameData) {
return scrapeGameStats(gameData.sportsReferenceGameId).catch((error) => {
Logger.error(`Failed to scrape game stats for ${gameData.sportsReferenceGameId}:`, {error});
logger.error(`Failed to scrape game stats for ${gameData.sportsReferenceGameId}:`, {error});
throw error;
});
} else {
Expand All @@ -96,9 +98,9 @@ const fn = async () => {

fs.writeFileSync(filename, JSON.stringify(yearData, null, 2));

Logger.success('Scraped game stats');
logger.success('Scraped game stats');
} catch (error) {
Logger.error('Failed to scrape game stats', {error});
logger.error('Failed to scrape game stats', {error});
} finally {
browser.close();
}
Expand Down
8 changes: 5 additions & 3 deletions scripts/archive/scrapeHeadCoachesAndBowlGames_WP.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import request from 'request-promise';

import {Logger} from '../lib/logger';

const logger = new Logger({isSentryEnabled: false});

const INPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../data/schedules');

const getHtmlForUrl = (url) => {
Expand All @@ -18,7 +20,7 @@ const getHtmlForUrl = (url) => {
});
};

Logger.info(`Fetching head coaches and bowl games.`);
logger.info(`Fetching head coaches and bowl games.`);

getHtmlForUrl(`https://en.wikipedia.org/wiki/List_of_Notre_Dame_Fighting_Irish_football_seasons`)
.then(($) => {
Expand Down Expand Up @@ -71,8 +73,8 @@ getHtmlForUrl(`https://en.wikipedia.org/wiki/List_of_Notre_Dame_Fighting_Irish_f
}
});

Logger.success('Fetched head coaches and bowl games');
logger.success('Fetched head coaches and bowl games');
})
.catch((error) => {
Logger.error(`Failed to fetch head coaches and bowl games:`, error.message);
logger.error(`Failed to fetch head coaches and bowl games:`, error.message);
});
12 changes: 7 additions & 5 deletions scripts/archive/scrapeSchedule_UND.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,18 @@ import RSVP from 'rsvp';
import {Logger} from '../../lib/logger';
import teamMappings from './teamMappings.json';

const logger = new Logger({isSentryEnabled: false});

if (process.argv.length !== 3) {
Logger.error('USAGE: node scrapeSchedule.js <output_file>');
logger.error('USAGE: node scrapeSchedule.js <output_file>');
process.exit(1);
}

/**
* Fetches the raw HTML schedule data for a given year.
*
* @param {number} year The year whose schedule to fetch.
* @return {Promise<Cheerio>} A Cheerio object containing the HTML schedule data.
* @return {Promise<cheerio.Root>} The HTML schedule data.
*/
const getHtmlScheduleDataForYear = (year) => {
return request({
Expand Down Expand Up @@ -149,15 +151,15 @@ _.forEach(years, (year) => {
});
})
.catch(function (error) {
Logger.error(`Error scraping ${year} schedule:`, {error});
logger.error(`Error scraping ${year} schedule:`, {error});
});
});

RSVP.hash(promises)
.then(function (result) {
fs.writeFileSync(process.argv[2], JSON.stringify(result, null, 2));
Logger.success(`Schedule written to ${process.argv[2]}!`);
logger.success(`Schedule written to ${process.argv[2]}!`);
})
.catch(function (error) {
Logger.fail('Failed to scrape schedule for all years:', {error});
logger.fail('Failed to scrape schedule for all years:', {error});
});
16 changes: 9 additions & 7 deletions scripts/archive/scrapeSchedule_jhowell.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import puppeteer from 'puppeteer';

import {Logger} from '../lib/logger';

const logger = new Logger({isSentryEnabled: false});

process.setMaxListeners(Infinity);

const ND_YEARS = _.range(1887, 2018).filter((year) => !_.includes([1890, 1891], year));
Expand Down Expand Up @@ -112,13 +114,13 @@ const scrapeNotreDameSchedule = async () => {
});

try {
Logger.info('Scraping Notre Dame schedule');
logger.info('Scraping Notre Dame schedule');

const ndSchedule = await scrapeNotreDameSchedule();

await browser.close();

Logger.success('Successfully scraped Notre Dame schedule');
logger.success('Successfully scraped Notre Dame schedule');

ND_YEARS.forEach((year) => {
if (year in ndSchedule) {
Expand All @@ -128,11 +130,11 @@ const scrapeNotreDameSchedule = async () => {

games.forEach((game, i) => {
if (game.isHomeGame !== ndSchedule[year][i].isHomeGame) {
Logger.error('HOME / AWAY MISMATCH', {year, opponentId: game.opponentId, index: i});
logger.error('HOME / AWAY MISMATCH', {year, opponentId: game.opponentId, index: i});
}

if (game.result !== ndSchedule[year][i].result) {
Logger.error('RESULT MISMATCH', {year, opponentId: game.opponentId, index: i});
logger.error('RESULT MISMATCH', {year, opponentId: game.opponentId, index: i});
}

if (
Expand All @@ -141,7 +143,7 @@ const scrapeNotreDameSchedule = async () => {
(game.score.away !== ndSchedule[year][i].score.home &&
game.score.away !== ndSchedule[year][i].score.away)
) {
Logger.error('SCORE MISMATCH', {
logger.error('SCORE MISMATCH', {
year,
opponentId: game.opponentId,
index: i,
Expand All @@ -153,8 +155,8 @@ const scrapeNotreDameSchedule = async () => {
}
});

Logger.success('Scraped Notre Dame schedule!');
logger.success('Scraped Notre Dame schedule!');
} catch (error) {
Logger.error('Failed to scraped Notre Dame schedule', {error});
logger.error('Failed to scraped Notre Dame schedule', {error});
}
})();
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import {Logger} from '../../../lib/logger';
import onePossessionNdCoachData from './data/ndCoachesOnePossessionGames.json';
import onePossessionGameData from './data/top25OnePossessionGames.json';

const logger = new Logger({isSentryEnabled: false});

_.forEach(onePossessionGameData, (data, teamName) => {
onePossessionGameData[teamName].teamName = teamName;
onePossessionGameData[teamName].percentageOfGamesWhichAreOnePossessionGames = (
Expand All @@ -24,17 +26,17 @@ const teamsSortedByPercentOfGamesWhichAreOnePossessionGames = _.sortBy(
'percentageOfGamesWhichAreOnePossessionGames'
);

Logger.newline(2);
logger.newline(2);

teamsSortedByPercentOfGamesWhichAreOnePossessionGames.forEach(
({teamName, top25Finishes, percentageOfGamesWhichAreOnePossessionGames}) => {
Logger.info(
logger.info(
`["${teamName}", ${top25Finishes}, "", "${percentageOfGamesWhichAreOnePossessionGames}%"],`
);
}
);

Logger.newline(2);
logger.newline(2);

const coachesSortedByPercentOfGamesWhichAreOnePossessionGames = _.sortBy(
onePossessionNdCoachData,
Expand All @@ -44,7 +46,7 @@ const coachesSortedByPercentOfGamesWhichAreOnePossessionGames = _.sortBy(
coachesSortedByPercentOfGamesWhichAreOnePossessionGames.forEach(
({name, totalGamesCount, percentageOfGamesWhichAreOnePossessionGames}) => {
if (totalGamesCount > 10 && name !== 'No coach') {
Logger.info(
logger.info(
`["${name}", ${totalGamesCount}, "${percentageOfGamesWhichAreOnePossessionGames}%"],`
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import _ from 'lodash';

import {Logger} from '../../../lib/logger';

const logger = new Logger({isSentryEnabled: false});

const INPUT_DATA_DIRECTORY = path.resolve(__dirname, '../../data/schedules');

const dataFilenames = fs.readdirSync(INPUT_DATA_DIRECTORY);
Expand All @@ -21,7 +23,7 @@ const getPercentage = (top, bottom) => {
return ((top * 100.0) / bottom).toFixed(1);
};

Logger.info('YEARS:');
logger.info('YEARS:');

dataFilenames.forEach((dataFilename) => {
const year = dataFilename.split('.')[0];
Expand Down Expand Up @@ -97,9 +99,9 @@ dataFilenames.forEach((dataFilename) => {
}
});

Logger.newline();
Logger.info(year);
Logger.info({
logger.newline();
logger.info(year);
logger.info({
...games[year],
record: `${games[year].onePossesssionWinsCount}-${games[year].onePossesssionLossesCount}-${games[year].onePossesssionTiesCount}`,
overallWinPercentage:
Expand All @@ -111,13 +113,13 @@ dataFilenames.forEach((dataFilename) => {
}
});

Logger.info(coaches);
Logger.info('\n\nCOACHES');
logger.info(coaches);
logger.info('\n\nCOACHES');

_.forEach(coachOrder, (coach) => {
const coachData = coaches[coach];

Logger.info(
logger.info(
coach,
coachData.onePossesssionGamesCount,
`${coachData.onePossesssionWinsCount}-${coachData.onePossesssionLossesCount}-${coachData.onePossesssionTiesCount}`,
Expand Down
Loading

0 comments on commit 1e07cb7

Please sign in to comment.