Support url based evaluation

README.md

@@ -6,10 +6,16 @@
 ```sh
 npx supported <path/to/node_module>
 npx supported <[array/of/node_modules]>
+// Generate token using https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token
+npx supported https://github.com/stefanpenner/supported
+npx supported https://github.com/stefanpenner/supported/tree/some-branch
+npx supported https://test.githubprivate.com/stefanpenner/supported -t $TOKEN
+npx supported supported --hostUrl=https://raw.githubusercontent.com/stefanpenner/supported/main/
 ```
 
-#### Optional Flags
+### Optional Flags
 
+#### current date
 The `--current-date` (`-c`) flag enables a form of limited time travel, and attempts to run
 the tools internal date calculations based on a specified date, rather then the
 current date.
@@ -26,6 +32,21 @@ Anything that `new Date(input)` parses, or if that fails it will assume to be a
 relative duration starting today parsed by
 [parse-duration@^1.0.0's own micro-syntax](https://github.com/jkroso/parse-duration#available-unit-types-are).
 
+#### hostURL
+The `--hostURL` flag enables a way to provide a valid URL which will return package.json, lock file and npmrc file if exists.
+some examples:
+
+* `--hostUrl=https://raw.githubusercontent.com/stefanpenner/supported/main/`, gets the above listed file from the provided URL.
+* `--hostUrl=https://${TOKEN}@raw.githubprivate.com/stefanpenner/supported/main/`, gets the above listed file from the private instance URL provided,
+private instance needs token, that must be passed as part of URL.
+
+#### token
+The `--token` (`-t`) is to pass the token generated to access the private instances of the github. This will enable this tool to evaluate
+the github private instance repositories. Generating a personal access token is explained in detail [here](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token).
+
+Example:
+`supported https://test.githubprivate.com/stefanpenner/supported -t $TOKEN`
+
 ### As a node module
 
 

lib/cli.js

@@ -9,6 +9,7 @@ const { displayResults } = require('../lib/output/cli-output');
 const { processPolicies } = require('../lib/project/multiple-projects');
 const { DEFAULT_SUPPORT_MESSAGE } = require('./output/messages');
 const { generateCsv } = require('../lib/output/csv-output');
+const { inputHasGithubPrivate } = require('./util');
 
 async function main(cli, { policyDetails, setupProjectFn }) {
   const projectPaths = handleInput(cli.input, process.cwd());
@@ -22,7 +23,7 @@ async function main(cli, { policyDetails, setupProjectFn }) {
     let result;
     let processed = false;
     try {
-      result = await processPolicies(projectPaths, setupProjectFn, spinner, currentDate);
+      result = await processPolicies(projectPaths, setupProjectFn, spinner, currentDate, cli.flags);
       if (result.isInSupportWindow === false) {
         process.exitCode = 1;
       }
@@ -87,6 +88,14 @@ async function run(
           type: 'string',
           alias: 'c',
         },
+        token: {
+          type: 'string',
+          alias: 't',
+          isRequired: inputHasGithubPrivate,
+        },
+        hostUrl: {
+          type: 'string',
+        },
       },
     }),
     { policyDetails, setupProjectFn },

lib/help.js

@@ -18,7 +18,12 @@ module.exports = (commandName = 'supported', packageLink = pkg.homepage) => {
     {cyan --supported, -s}    outputs detailed report of supported packages only
     {cyan --csv}              outputs csv file in the project path
     {cyan --current-date, -c} optional current date to use when calculating support
+    {cyan --token, -t}        token to access raw file from github private instance
+    {cyan --hostUrl}          URL endpoint that returns package.json, yarn.lock and npmrc files.
   {bold Examples}
     {gray $} {cyan  ${commandName} ./path/to/project/}
+    {gray $} {cyan  ${commandName} https://github.com/stefanpenner/supported}
+    {gray $} {cyan  ${commandName} https://test.githubprivate.com/stefanpenner/supported -t $TOKEN}
+    {gray $} {cyan  ${commandName} supported --hostUrl=https://raw.githubusercontent.com/stefanpenner/supported/main/}
 `;
 };

lib/lts/ember-lts.json

@@ -1,14 +1,14 @@
 {
-  "3.16.*": {
-    "versionRange": "3.16.*",
-    "start_date": "2020-03-04T00:00:00.000Z",
-    "maintenance_start_date": "2020-11-11T00:00:00.000Z",
-    "end_date": "2021-03-17T00:00:00.000Z"
-  },
   "3.20.*":	{
     "versionRange": ">=3.20.*",
     "start_date": "2020-08-24T00:00:00.000Z",
     "maintenance_start_date": "2021-05-03T00:00:00.000Z",
     "end_date": "2021-09-06T00:00:00.000Z"
+  },
+  "3.24.*":	{
+    "versionRange": ">=3.24.*",
+    "start_date": "2021-02-25T00:00:00.000Z",
+    "maintenance_start_date": "2021-11-04T00:00:00.000Z",
+    "end_date": "2022-03-10T00:00:00.000Z"
   }
 }

lib/project/multiple-projects.js

@@ -10,7 +10,7 @@ const { ProgressLogger } = require('../util');
 const DEFAULT_SETUP_FILE = './setup-project';
 
 module.exports.processPolicies = processPolicies;
-async function processPolicies(projectPaths, setupProjectFn, spinner, today) {
+async function processPolicies(projectPaths, setupProjectFn, spinner, today, flags) {
   const setupProject = setupProjectFn ? setupProjectFn : require(DEFAULT_SETUP_FILE);
   let result = {
     isInSupportWindow: true,
@@ -29,7 +29,7 @@ async function processPolicies(projectPaths, setupProjectFn, spinner, today) {
   for (const projectPath of projectPaths) {
     work.push(
       queue.add(async () => {
-        let { dependenciesToCheck, pkg } = await setupProject(projectPath);
+        let { dependenciesToCheck, pkg } = await setupProject(projectPath, flags);
         progressLogger.updateTotalDepCount(dependenciesToCheck.length);
         let auditResult = await isInSupportWindow(
           dependenciesToCheck,

lib/project/setup-project.js

@@ -3,32 +3,65 @@
 const semverCoerce = require('semver/functions/coerce');
 const YarnLockfile = require('@yarnpkg/lockfile');
 const npa = require('npm-package-arg');
+const ini = require('ini');
 const fs = require('fs');
 const path = require('path');
 const debug = require('debug')('supported:project');
+const { setupProjectPath } = require('../read-from-url');
 
 const npmConfig = require('../npm/config');
 
-module.exports = async function setupProject(projectRoot) {
-  const config = await npmConfig(projectRoot); // kinda slow, TODO: re-implement as standalone lib
-  // const { policies } = options;
-  const pkgPath = `${projectRoot}/package.json`;
-  if (!fs.existsSync(pkgPath)) {
-    throw new Error(`${pkgPath} does not exist, are you sure this is a valid package?`);
-  }
-  if (!fs.statSync(pkgPath).isFile()) {
-    throw new Error(`${pkgPath} is not a file, are you sure this is a valid package?`);
+module.exports = async function setupProject(projectRoot, flags = {}) {
+  let projectInfo = {};
+  if (!fs.existsSync(projectRoot)) {
+    projectInfo = await setupProjectPath(projectRoot, {
+      token: flags.token,
+      hostUrl: flags.hostUrl,
+    });
+    // if project root is a URL then use homedir as root
+    projectRoot = require('os').homedir();
   }
-  const file = fs.readFileSync(pkgPath, 'utf-8');
-  let pkg;
-  try {
-    pkg = JSON.parse(file);
-  } catch (e) {
-    throw new Error(`${pkgPath} is not a valid JSON file, are you sure this is a valid package?`);
+
+  let config = await npmConfig(projectRoot); // kinda slow, TODO: re-implement as standalone lib
+  let pkg = {};
+  let lockfileContent = '';
+  if (projectInfo.packageJSON) {
+    pkg = projectInfo.packageJSON;
+    lockfileContent = projectInfo['yarn.lock'];
+    if (projectInfo['.npmrc']) {
+      try {
+        let localConfig = ini.parse(projectInfo['.npmrc']);
+        config = {
+          config,
+          ...localConfig,
+        };
+      } catch (e) {
+        throw new Error(
+          `Couldn't parse the npmrc file, are you sure project URL has a valid npmrc?\n ${e}`,
+        );
+      }
+    }
+  } else {
+    const pkgPath = `${projectRoot}/package.json`;
+    if (!fs.existsSync(pkgPath)) {
+      throw new Error(`${pkgPath} does not exist, are you sure this is a valid package?`);
+    }
+    if (!fs.statSync(pkgPath).isFile()) {
+      throw new Error(`${pkgPath} is not a file, are you sure this is a valid package?`);
+    }
+    const file = fs.readFileSync(pkgPath, 'utf-8');
+    try {
+      pkg = JSON.parse(file);
+    } catch (e) {
+      throw new Error(`${pkgPath} is not a valid JSON file, are you sure this is a valid package?`);
+    }
+    const lockfilePath = path.join(projectRoot, 'yarn.lock');
+    lockfileContent = fs.readFileSync(lockfilePath, 'utf-8');
   }
+  // const { policies } = options;
   const lockfilePath = path.join(projectRoot, 'yarn.lock');
   // TODO: npm support
-  const { object: lockfile } = YarnLockfile.parse(fs.readFileSync(lockfilePath, 'utf-8'));
+  const { object: lockfile } = YarnLockfile.parse(lockfileContent);
 
   const dependenciesToCheck = [];
   if (pkg.dependencies) {

lib/read-from-url.js

@@ -0,0 +1,85 @@
+'use strict';
+const fetch = require('minipass-fetch');
+const debug = require('debug')('supported:read-from-url');
+const { default: PQueue } = require('p-queue');
+const allSettled = require('promise.allsettled');
+const os = require('os');
+const { getFetchUrl } = require('./util');
+const chalk = require('chalk');
+
+async function setupProjectPath(_url, options = {}) {
+  let url = getFetchUrl(_url, options);
+
+  const work = [];
+  const queue = new PQueue({
+    concurrency: os.cpus().length,
+  });
+  let result = Object.create(null);
+  let packageJSON;
+  // check if the code moved to main or still using master
+  // https://github.com/SparshithNR/doc-tester/blob/master/package.json
+  // https://github.com/stefanpenner/supported/blob/main/package.json
+  try {
+    packageJSON = await runFetch(url + 'package.json', true);
+  } catch (e) {
+    if (e.code === 404) {
+      url = url.replace('main', 'master');
+      packageJSON = await runFetch(url + 'package.json', true);
+    } else {
+      throw e;
+    }
+  }
+  for (const fileName of ['yarn.lock', '.npmrc']) {
+    work.push(
+      queue.add(async () => {
+        const requestURL = url + fileName;
+        result[fileName] = await runFetch(requestURL);
+      }),
+    );
+  }
+  await queue.onIdle();
+  for (const settled of await allSettled(work)) {
+    if (settled.status === 'rejected') {
+      throw settled.reason;
+    }
+  }
+
+  debug(`packageJSON url: ${url.toString()}`);
+
+  return {
+    packageJSON,
+    ...result,
+  };
+}
+
+async function runFetch(requestURL, isJson) {
+  let response;
+  try {
+    response = await fetch(requestURL);
+  } catch (e) {
+    let error = new Error(chalk`{red Couldn't reach server, please check the URL provided.}
+    ${e.message}`);
+    throw error;
+  }
+  if (response.status === 200) {
+    if (isJson) {
+      return response.json();
+    }
+    return response.text();
+  } else if (response.status === 404) {
+    if (!requestURL.includes('.npmrc')) {
+      const text = await response.buffer();
+      const e = new Error(`[http.status=${response.status}] url:${requestURL} : error: ${text}`);
+      e.code = response.status;
+      throw e;
+    } else {
+      debug(`Fetch failed for .npmrc , ${await response.buffer()}`);
+    }
+  } else {
+    throw new Error(`[http.status=${response.status}] url:${requestURL}`);
+  }
+}
+
+module.exports = {
+  setupProjectPath,
+};

lib/util.js

@@ -14,6 +14,12 @@ exports.MS_IN_QTR = MS_IN_QTR;
 // expiring soon if the package expires within 4 qtrs
 const THRESHOLD_QTRS = 5;
 
+// github url constants
+const RAW_CONTENT_HOST = 'raw.githubusercontent.com';
+const GITHUB_PRIVATE = 'githubprivate';
+const GITHUB = 'github.com';
+const MISSING_TOKEN = chalk`{red Private instances of github needs token. To generate token follow https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token}`;
+
 /**
  *
  * @param {int}  timeDiff duration between latest version and resolved version release date
@@ -232,3 +238,55 @@ module.exports.handleInput = function (input, cwd) {
     return input;
   }
 };
+
+/**
+ * Generates the fetch URL to get package.json and other required files.
+ */
+module.exports.getFetchUrl = function (_url, options = {}) {
+  let { token, hostUrl } = options;
+  _url = hostUrl || _url;
+  _url += _url.endsWith('/') ? '' : '/';
+  if (hostUrl) {
+    return _url;
+  }
+  let url = new URL(_url);
+  let branch = url.pathname.indexOf('/tree/');
+  let hostname = url.hostname;
+  let pathname = url.pathname;
+  if (hostname.includes(GITHUB)) {
+    hostname = RAW_CONTENT_HOST;
+    if (branch == -1) {
+      pathname += 'main/';
+    }
+  } else if (hostname.includes(GITHUB_PRIVATE)) {
+    if (!token) {
+      throw new Error(MISSING_TOKEN);
+    }
+    hostname = `${token}@raw.${hostname}`;
+    if (branch == -1) {
+      pathname += 'master/';
+    }
+  }
+  if (branch !== -1) {
+    pathname = pathname.replace('/tree/', '/');
+  }
+  let currentURL = url.toString();
+  currentURL = currentURL.replace(url.hostname, hostname);
+  currentURL = currentURL.replace(url.pathname, pathname);
+  currentURL += currentURL.endsWith('/') ? '' : '/';
+  return currentURL;
+};
+
+/**
+ * Returns true if the given list of input has private instance, and token is provided.
+ */
+module.exports.inputHasGithubPrivate = function (flags, input) {
+  input = Array.isArray(input) ? input : [input];
+  let isPrivate = input.some(inputUrl => {
+    return inputUrl.includes(GITHUB_PRIVATE);
+  });
+  if (isPrivate && !flags.token) {
+    console.log(MISSING_TOKEN);
+  }
+  return isPrivate;
+};

package.json

@@ -50,7 +50,7 @@
     "cli-table": "^0.3.4",
     "debug": "^4.3.1",
     "execa": "^5.0.0",
-    "ini": "^1.3.7",
+    "ini": "^2.0.0",
     "json2csv": "^5.0.6",
     "koa": "^2.13.0",
     "meow": "^8.0.0",