Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(publish): watch mode #24

Open
wants to merge 24 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,17 @@ _Publish mode_ helps replicate the production environment in your development se
Any changes you make to the _Dependency package_ will be reflected in the `node_modules` directory of the _Consuming package_.

> **Note:** If the _Dependency package_ emits new files, you'll need to re-run `npx link publish <dependency-package-path>` to create new hard links.


#### Watch mode

To automatically re-link a package in publish mode whenever a change is made, use the `--watch` flag:

```sh
npx link publish --watch <dependency-package-path>
```

Hard links do not support directories, so files must be individually linked. Watch mode addresses this limitation by automatically linking new files added to the _Dependency package_ so they appear in the _Consuming package_.

### Configuration file

Create a `link.config.json` (or `link.config.js`) configuration file at the root of the _Consuming package_ to automatically setup links to multiple _Dependency packages_.
Expand Down
3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
},
"devDependencies": {
"@types/cmd-shim": "^5.0.2",
"@types/glob-to-regexp": "^0.4.4",
"@types/node": "^20.14.14",
"@types/npm-packlist": "^7.0.3",
"@types/npmcli__package-json": "^4.0.4",
Expand All @@ -38,12 +39,14 @@
"execa": "^8.0.1",
"fs-fixture": "^2.4.0",
"get-node": "^15.0.1",
"glob-to-regexp": "^0.4.1",
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Noticed that this dependency is marked as archived/read-only, while a more popular alternative https://github.com/isaacs/minimatch is still actively maintained (and apparently used by npm internals).

Was there some reason you chose glob-to-regex instead?

"kolorist": "^1.8.0",
"lintroll": "^1.7.1",
"manten": "^1.3.0",
"npm-packlist": "^8.0.2",
"outdent": "^0.8.0",
"pkgroll": "^2.4.2",
"throttleit": "^2.1.0",
"tsx": "^4.16.5",
"typescript": "^5.5.4"
}
Expand Down
25 changes: 25 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,11 @@ import { publishCommand, publishHandler } from './commands/publish/index.js';
},
);
} else if (argv.command === 'publish') {
await publishHandler(cwdProjectPath, argv._);
await publishHandler(
cwdProjectPath,
argv._,
argv.flags,
);
}
})().catch((error) => {
console.error('Error:', error.message);
Expand Down
9 changes: 5 additions & 4 deletions src/commands/publish/hardlink-package.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ export const hardlinkPackage = async (
linkPath: string,
absoluteLinkPackagePath: string,
packageJson: PackageJsonWithName,
publishFilesPromise: string[] | Promise<string[]> = getNpmPacklist(
absoluteLinkPackagePath,
packageJson,
),
) => {
const [oldPublishFiles, publishFiles] = await Promise.all([
getNpmPacklist(
Expand All @@ -21,10 +25,7 @@ export const hardlinkPackage = async (
*/
packageJson,
),
getNpmPacklist(
absoluteLinkPackagePath,
packageJson,
),
publishFilesPromise,
]);

console.log(`Linking ${magenta(packageJson.name)} in publish mode:`);
Expand Down
12 changes: 7 additions & 5 deletions src/commands/publish/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ export const publishCommand = command({
name: 'publish',
parameters: ['<package paths...>'],
flags: {
// watch: {
// type: Boolean,
// alias: 'w',
// description: 'Watch for changes in the package and automatically relink',
// },
watch: {
type: Boolean,
alias: 'w',
description: 'Watch for changes in the package and automatically relink',
},
},
help: {
description: 'Link a package to simulate an environment similar to `npm install`',
Expand All @@ -19,13 +19,15 @@ export const publishCommand = command({
export const publishHandler = async (
cwdProjectPath: string,
packagePaths: string[],
flags: { watch?: boolean },
) => {
if (packagePaths.length > 0) {
await Promise.all(
packagePaths.map(
linkPackagePath => linkPublishMode(
cwdProjectPath,
linkPackagePath,
flags.watch,
),
),
);
Expand Down
86 changes: 71 additions & 15 deletions src/commands/publish/link-publish-mode.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
import path from 'path';
import fs from 'fs/promises';
import path from 'node:path';
import fs from 'node:fs/promises';
import outdent from 'outdent';
import { magenta, bold, dim } from 'kolorist';
import throttle from 'throttleit';
import globToRegexp from 'glob-to-regexp';
import {
magenta, cyan, bold, dim, yellow,
} from 'kolorist';
import { readPackageJson } from '../../utils/read-package-json.js';
import { getNpmPacklist } from '../../utils/get-npm-packlist.js';
import { cwdPath } from '../../utils/cwd-path.js';
import { getPrettyTime } from '../../utils/get-pretty-time.js';
import { hardlinkPackage } from './hardlink-package.js';

const isValidSetup = async (
Expand All @@ -27,6 +34,7 @@ const isValidSetup = async (
export const linkPublishMode = async (
basePackagePath: string,
linkPackagePath: string,
watchMode?: boolean,
) => {
const absoluteLinkPackagePath = path.resolve(basePackagePath, linkPackagePath);
const packageJson = await readPackageJson(absoluteLinkPackagePath);
Expand Down Expand Up @@ -54,20 +62,68 @@ export const linkPublishMode = async (
return;
}

/**
* If it's a symlink, make sure it's in the node_modules directory of the base package.
* e.g. This could happen with pnpm
*
* If it's not, it might be a development directory and we don't want to overwrite it.
*/
const linkPathReal = await fs.realpath(linkPath);
if (!linkPathReal.startsWith(expectedPrefix)) {
return;
}

await hardlinkPackage(
const throttledHardlinkPackage = throttle(hardlinkPackage, 500);
await throttledHardlinkPackage(
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not immediately clear to me why you chose to throttle this function. Maybe a code comment would be a good idea here.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also was curious so I investigated a bit the difference between throttleit and p-throttle. I think throttleit is appropriate here 👍🏼, since I assume we want to discard any but the last invocation of hardlinkPackage in an interval

linkPath,
absoluteLinkPackagePath,
packageJson,
);

if (watchMode) {
const globOptions = {
globstar: true,
extended: true,
};

/**
* npm-packlist ignore list:
* https://github.com/npm/npm-packlist/blob/v8.0.2/lib/index.js#L15-L38
*/
const ignoreFiles = [
// Files
'**/{npm-debug.log,*.orig,package-lock.json,yarn.lock,pnpm-lock.yaml}',

// Folders
'**/node_modules/**',

// Hidden files
'**/.{_*,*.swp,DS_Store,gitignore,npmrc,npmignore,lock-wscript,.wafpickle-*}',

// Hidden folders
'**/.{_*,git,svn,hg,CVS}/**',
].map(glob => globToRegexp(glob, globOptions));

const watcher = fs.watch(
absoluteLinkPackagePath,
{ recursive: true },
);

for await (const { eventType, filename } of watcher) {
if (!filename) {
continue;
}

const shouldIgnore = ignoreFiles.some(ignoreFile => ignoreFile.test(filename));
if (shouldIgnore) {
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably doesn't matter too much, but one thing you could consider if you're sticking with globToRegexp is joining the regexes together then calling test just once rather than in a loop:

diff --git a/src/commands/publish/link-publish-mode.ts b/src/commands/publish/link-publish-mode.ts
index e0b535c..69caad3 100644
--- a/src/commands/publish/link-publish-mode.ts
+++ b/src/commands/publish/link-publish-mode.ts
@@ -79,7 +79,7 @@ export const linkPublishMode = async (
 		 * npm-packlist ignore list:
 		 * https://github.com/npm/npm-packlist/blob/v8.0.2/lib/index.js#L15-L38
 		 */
-		const ignoreFiles = [
+		const ignoreFiles = new RegExp([
 			// Files
 			'**/{npm-debug.log,*.orig,package-lock.json,yarn.lock,pnpm-lock.yaml}',
 
@@ -91,7 +91,8 @@ export const linkPublishMode = async (
 
 			// Hidden folders
 			'**/.{_*,git,svn,hg,CVS}/**',
-		].map(glob => globToRegexp(glob, globOptions));
+		].map(glob => globToRegexp(glob, globOptions).source)
+		 .join('|'));
 
 		const watcher = fs.watch(
 			absoluteLinkPackagePath,
@@ -103,7 +104,7 @@ export const linkPublishMode = async (
 				continue;
 			}
 
-			const shouldIgnore = ignoreFiles.some(ignoreFile => ignoreFile.test(filename));
+			const shouldIgnore = ignoreFiles.test(filename);
 			if (shouldIgnore) {
 				continue;
 			}

continue;
}

const publishFiles = await getNpmPacklist(
absoluteLinkPackagePath,
packageJson,
);

if (!publishFiles.includes(filename)) {
continue;
}

console.log(`\n${dim(getPrettyTime())}`, 'Detected', yellow(eventType), 'in', `${cyan(cwdPath(path.join(absoluteLinkPackagePath, filename)))}\n`);
await throttledHardlinkPackage(
linkPath,
absoluteLinkPackagePath,
packageJson,
publishFiles,
);
}
}
};
9 changes: 9 additions & 0 deletions src/utils/get-pretty-time.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
export const getPrettyTime = () => (new Date()).toLocaleTimeString(
undefined,
{
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
hour12: true,
},
);
45 changes: 38 additions & 7 deletions tests/specs/publish.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,20 @@ import { testSuite, expect } from 'manten';
import { execa } from 'execa';
import { createFixture } from 'fs-fixture';
import { link } from '../utils/link.js';
import { streamWaitFor } from '../utils/stream-wait-for.js';
import { npmPack } from '../utils/npm-pack.js';

export default testSuite(({ describe }, nodePath: string) => {
describe('publish mode', ({ test }) => {
test('hard links', async () => {
await using fixture = await createFixture('./tests/fixtures/');
describe('publish mode', async ({ test }) => {
await using fixture = await createFixture('./tests/fixtures/');

const packageFilesPath = fixture.getPath('package-files');
const statOriginalFile = await fs.stat(fixture.getPath('package-files/package.json'));
const tarballPath = await npmPack(packageFilesPath);
const entryPackagePath = fixture.getPath('package-entry');
// Prepare tarball to install
const packageFilesPath = fixture.getPath('package-files');
const statOriginalFile = await fs.stat(fixture.getPath('package-files/package.json'));
const tarballPath = await npmPack(packageFilesPath);
const entryPackagePath = fixture.getPath('package-entry');

await test('hard links', async () => {
await execa('npm', [
'install',
'--no-save',
Expand All @@ -40,5 +42,34 @@ export default testSuite(({ describe }, nodePath: string) => {
const statAfterLink = await fs.stat(path.join(entryPackagePath, 'node_modules/package-files/package.json'));
expect(statAfterLink.ino).toBe(statOriginalFile.ino);
});

await test('watch mode', async () => {
const watchMode = link([
'publish',
'--watch',
packageFilesPath,
], {
cwd: entryPackagePath,
nodePath,
});

// Wait for initial hardlink
await streamWaitFor(watchMode.stdout!, '✔');

// Should trigger watch because lib is in files
fixture.writeFile('package-files/lib/file-a.js', 'file-a');
await streamWaitFor(watchMode.stdout!, 'lib/file-a.js');

// Should not trigger watch because it's not in lib
await fixture.writeFile('package-files/file-b.js', 'file-b');
await expect(
() => streamWaitFor(watchMode.stdout!, 'file-b.js', 1000),
).rejects.toThrow('Timeout');

watchMode.kill();
await watchMode;
}, {
timeout: 5000,
});
});
});
Loading
Loading