WIP: Rewrite symlink and module logic

jest.config.js

@@ -7,6 +7,7 @@ module.exports = {
   "coverageThreshold": {
     "global": { lines: 30 }
   },
+  "testEnvironment": "node",
   "moduleFileExtensions": ["ts", "tsx", "js"],
   "transform": { ".(ts|tsx)": "<rootDir>/node_modules/ts-jest/preprocessor.js" },
   "testRegex": "(\\.(test|spec))\\.(ts|tsx|js)$",

package.json

@@ -19,21 +19,25 @@
     "preversion": "yarn build && yarn test:dist"
   },
   "dependencies": {
+    "@types/lodash": "^4.14.117",
+    "@types/resolve": "^0.0.8",
     "archiver": "^2.0.0",
     "bluebird": "^3.5.0",
     "chalk": "^2.1.0",
     "fs-extra": "^4.0.1",
     "get-folder-size": "^1.0.0",
     "is-stream": "~1.1.0",
     "js-yaml": "^3.9.1",
+    "lodash": "^4.17.11",
     "lutils": "^2.4.0",
     "minimatch": "^3.0.4",
     "mkdirp": "~0.5.1",
+    "resolve": "^1.8.1",
     "resolve-pkg": "^1.0.0",
     "semver": "^5.4.1",
     "source-map-support": "^0.4.15",
-    "ts-node": "^3.3.0",
-    "typescript": "^2.3.0",
+    "ts-node": "^7.0.1",
+    "typescript": "^3.1.3",
     "walker": "^1.0.7"
   },
   "devDependencies": {
@@ -44,7 +48,7 @@
     "@types/fs-extra": "^4.0.0",
     "@types/fs-promise": "^1.0.3",
     "@types/graceful-fs": "^2.0.29",
-    "@types/jest": "^20.0.6",
+    "@types/jest": "^22.0.0",
     "@types/js-yaml": "^3.9.0",
     "@types/minimatch": "^2.0.29",
     "@types/mkdirp": "^0.5.0",
@@ -53,9 +57,9 @@
     "@types/source-map-support": "^0.4.0",
     "@types/typescript": "^2.0.0",
     "@types/uglify-js": "^2.6.29",
-    "jest": "^20.0.4",
-    "ts-jest": "^20.0.10",
-    "tslint": "^5.6.0",
+    "jest": "^22.0.0",
+    "ts-jest": "^22.0.0",
+    "tslint": "^5.11.0",
     "tslint-config-temando": "^1.2.0",
     "uglify-js": "3"
   },

src/ModuleBundler.ts

@@ -1,14 +1,16 @@
+import { Archiver } from 'archiver';
 import * as Bluebird from 'bluebird';
-import { join, sep } from 'path';
+import { existsSync } from 'fs-extra';
+import { basename, join, sep } from 'path';
 import * as resolvePackage from 'resolve-pkg';
 
-import { Archiver } from 'archiver';
-import { existsSync } from 'fs-extra';
 import { Logger } from './lib/Logger';
+import { readPath } from './lib/readPath';
 import { handleFile } from './lib/utils';
-import { findSymlinks, Walker } from './lib/Walker';
 import { UglifyTransform } from './transforms/Uglify';
 
+import resolve = require('resolve');
+
 export interface IModule {
   name: string;
   packagePath: string;
@@ -48,76 +50,59 @@ export class ModuleBundler {
   async bundle ({ include = [], exclude = [], deepExclude = [] }: {
     include?: string[], exclude?: string[], deepExclude?: string[],
   }) {
-    const links = await findSymlinks(join(this.servicePath, 'node_modules'));
-
     this.modules = this.resolveDependencies(
       this.servicePath,
-      { include, exclude, deepExclude, links },
-    );
+      { include, exclude, deepExclude },
+      );
 
     const transforms = this.resolveTransforms();
 
     const readModule = async ({ packagePath, packageDir, relativePath, packageJson }) => {
-      const filter = (dirPath, stats) => {
-        const { linkedPath, link } = this.resolveSymlinkPath(dirPath, links);
-
-        let testPackagePath = packagePath;
-
-        if (linkedPath) {
-          dirPath = linkedPath;
-          testPackagePath = link;
-        }
+      const basePath = `${packagePath}${sep}`;
+
+      await readPath(basePath, {
+        onFileFilter: ({ filePath }) => {
+          if (!deepExclude.length) { return false; }
+
+          // This pulls ['node_modules', 'pack'] out of
+          // .../node_modules/package/node_modules/pack
+          const endParts = filePath
+            .split(packagePath)[1]
+            .split('/')
+            .slice(-2);
+
+          // When a directory is a package and matches a deep exclude pattern
+          // Then skip it
+          if (
+            endParts[0] === 'node_modules' &&
+            deepExclude.indexOf(endParts[1]) !== -1
+          ) {
+            return true;
+          }
 
-        if (!dirPath || !testPackagePath) { return true; }
-
-        // If there are no deep exclusions, then there is no more filtering.
-        if (!deepExclude.length) {
-          return true;
-        }
-
-        // This pulls ['node_modules', 'pack'] out of
-        // .../node_modules/package/node_modules/pack
-        const endParts = dirPath.split(testPackagePath)[1].split('/').slice(-2);
-
-        // When a directory is a package and matches a deep exclude pattern
-        // Then skip it
-        if (
-          endParts[0] === 'node_modules' &&
-          deepExclude.indexOf(endParts[1]) !== -1
-        ) {
           return false;
-        }
-
-        return true;
-      };
-
-      const onFile = async (filePath: string, stats) => {
-        let relPath;
-
-        const { relLinkedPath } = this.resolveSymlinkPath(filePath, links);
-
-        if (relLinkedPath) { relPath = join(relativePath, relLinkedPath); }
-
-        if (!relPath) {
-          relPath = filePath.substr(filePath.indexOf(relativePath));
-        }
-
-        relPath = relPath.replace(/^\/|\/$/g, '');
-
-        await handleFile({
-          filePath,
-          relPath,
-          transforms,
-          transformExtensions: ['js', 'jsx'],
-          useSourceMaps: false,
-          archive: this.archive,
-        });
-      };
-
-      await new Walker(`${packagePath}${sep}`)
-        .filter(filter)
-        .file(onFile)
-        .end();
+        },
+        onFile: ({ filePath, previousPaths }) => {
+          console.log({ previousPaths });
+          const relPath = join(
+            'node_modules',
+            basename(packagePath),
+            ...previousPaths.slice(1).map((path) => basename(path)),
+            basename(filePath),
+          );
+
+          console.dir({ packageDir, packagePath, relPath, filePath }, { colors: true });
+
+          return handleFile({
+            filePath,
+            relPath,
+            transforms,
+            transformExtensions: ['js', 'jsx'],
+            useSourceMaps: false,
+            archive: this.archive,
+          });
+        },
+      });
 
       return this.logger.module(({ filePath: relativePath, realPath: packagePath, packageJson }));
     };
@@ -141,35 +126,12 @@ export class ModuleBundler {
     return transforms;
   }
 
-  private resolveSymlinkPath (filePath: string, links: Map<string, string>): {
-    linkedPath?: string,
-    link?: string, real?: string,
-    relLinkedPath?: string,
-  } {
-    const items = Array.from(links.entries()).reverse();
-
-    // Get a relPath from using a matching symlink
-    for (const [real, link] of items) {
-      if (filePath.startsWith(real)) {
-        const relLinkedPath = filePath.slice(real.length);
-
-        return {
-          real, link,
-          relLinkedPath,
-          linkedPath: join(link, relLinkedPath),
-        };
-      }
-    }
-
-    return {};
-  }
-
   /**
    * Resolves a package's dependencies to an array of paths.
    */
   private resolveDependencies (
     initialPackageDir,
-    { include = [], exclude = [], deepExclude = [], links = new Map() } = {},
+    { include = [], exclude = [], deepExclude = [] } = {},
   ): IModule[] {
     const resolvedDeps: IModule[] = [];
     const cache: Set<string> = new Set();
@@ -198,11 +160,21 @@ export class ModuleBundler {
         // Skips on include mis-matches, if set
         if (_include.length && _include.indexOf(packageName) < 0) { return; }
 
-        let nextPackagePath = resolvePackage(packageName, { cwd: packageDir });
-        if (!nextPackagePath) { return; }
+        const nextPackagePathOld = resolvePackage(packageName, { cwd: packageDir });
+        console.log({ nextPackagePathOld });
+        let main: string;
+        let nextPackagePath = resolve.sync(packageName, {
+          basedir: packageDir,
+          preserveSymlinks: true,
+          packageFilter (pkgJson) {
+            main = pkgJson.main || 'index.js';
+          },
+        });
 
-        const link = links.get(nextPackagePath);
-        if (link) { nextPackagePath = link; }
+        nextPackagePath = nextPackagePath.slice(0, -main.length - 1);
+
+        console.log({ nextPackagePathOld, nextPackagePath, main });
+        if (!nextPackagePath) { return; }
 
         const relativePath = join('node_modules', nextPackagePath.split(separator).slice(1).join(separator));
 

src/ServerlessBuildPlugin.ts

@@ -5,6 +5,7 @@ import { copy, createWriteStream, emptyDir, ensureDir } from 'fs-extra';
 import { clone, isArray, merge } from 'lutils';
 import * as path from 'path';
 import * as semver from 'semver';
+
 import { defaultConfig, IPluginConfig } from './config';
 import { FileBuild } from './FileBuild';
 import { Logger } from './lib/Logger';
@@ -87,7 +88,7 @@ export class ServerlessBuildPlugin {
 
     const functionSelection = this.config.f || this.config.function;
 
-    let selectedFunctions = isArray(functionSelection)
+    let selectedFunctions: string[] = isArray(functionSelection)
       ? functionSelection
       : [functionSelection];
 

src/__tests__/project/bSym

@@ -0,0 +1 @@
+b

src/__tests__/project/c/projectSym

@@ -0,0 +1 @@
+../

src/lib/Walker.ts

@@ -1,5 +1,5 @@
 import * as Bluebird from 'bluebird';
-import { lstat, readdir, realpath } from 'fs-extra';
+import { existsSync, lstat, readdir, realpath } from 'fs-extra';
 import { join } from 'path';
 import * as createWalker from 'walker';
 
@@ -8,8 +8,8 @@ export class Walker {
   pending: any[] = [];
   symlinkRoots: Set<string> = new Set();
 
-  constructor (directory) {
-    this.walker = createWalker(directory);
+  constructor (startDirectory: string) {
+    this.walker = createWalker(startDirectory);
   }
 
   filter (fn) {
@@ -57,6 +57,8 @@ export async function findSymlinks (dirPath, maxDepth = 2) {
 
     --depth;
 
+    if (!existsSync(dir)) { return; }
+
     const stats = await lstat(dir);
 
     if (stats.isSymbolicLink()) {

src/lib/__tests__/__snapshots__/readPath.test.ts.snap

@@ -0,0 +1,52 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`can filter out paths matching /c/ 1`] = `
+Array [
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/bSym/d",
+]
+`;
+
+exports[`can traverse circular symlinks up to depth 1`] = `
+Array [
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/bSym/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/c/projectSym/bSym/d",
+]
+`;
+
+exports[`reads regular file nested in 1 folder 1`] = `
+Array [
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/a",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/b/d",
+  "/home/sam/W/serverless-build-plugin/src/__tests__/project/bSym/d",
+]
+`;