feat: pages for npm cached packages

.babelrc

@@ -1,3 +1,6 @@
 {
-  "presets": [["@verdaccio", {"typescript": true}]]
+  "presets": [["@verdaccio", {"typescript": true}]],
+  "plugins": [
+    ["@babel/plugin-proposal-decorators", { "legacy":true }]
+  ]
 }

jest.config.js

@@ -1,5 +1,4 @@
 /* eslint comma-dangle: 0 */
-
 module.exports = {
   name: 'local-storage-jest',
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],

package.json

@@ -47,6 +47,7 @@
     "jest": "24.5.0",
     "minimatch": "3.0.4",
     "prettier": "1.16.4",
+    "promise-cache-decorator": "0.3.14",
     "rmdir-sync": "1.0.1",
     "standard-version": "5.0.2",
     "typescript": "3.2.1"

src/local-database.ts

@@ -4,11 +4,13 @@ import Path from 'path';
 // $FlowFixMe
 import async from 'async';
 import mkdirp from 'mkdirp';
+import cached from 'promise-cache-decorator';
 
 import LocalDriver, { noSuchFile } from './local-fs';
 import { loadPrivatePackages } from './pkg-utils';
 
 import { IPackageStorage, IPluginStorage, StorageList, LocalStorage, Logger, Config, Callback, PackageAccess } from '@verdaccio/types';
+import { findPackages } from "./utils";
 
 const DEPRECATED_DB_NAME: string = '.sinopia-db.json';
 const DB_NAME: string = '.verdaccio-db.json';
@@ -245,6 +247,18 @@ class LocalDatabase implements IPluginStorage<{}> {
     return new LocalDriver(packageStoragePath, this.logger);
   }
 
+  /**
+   * Get list of all chached package names with all vrsions
+   */
+  @cached({
+    id: 'all-local-packages-list',
+    type: 'age',
+    maxAge: 60 * 1000, // TTL = 1 minute
+  })
+  getPackagesAll() {
+    return findPackages(Path.dirname(this.path));
+  }
+
   /**
    * Verify the right local storage location.
    * @param {String} path

src/utils.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import _ from 'lodash';
 import path from 'path';
 
-export function getFileStats(packagePath: string): any {
+async function getFileStats(packagePath: string): Promise<any> {
   return new Promise((resolve, reject) => {
     fs.stat(packagePath, (err, stats) => {
       if (_.isNil(err) === false) {
@@ -13,7 +13,7 @@ export function getFileStats(packagePath: string): any {
   });
 }
 
-export function readDirectory(packagePath: string): Promise<any> {
+async function readDirectory(packagePath: string): Promise<any> {
   return new Promise((resolve, reject) => {
     fs.readdir(packagePath, (err, scopedPackages) => {
       if (_.isNil(err) === false) {
@@ -29,44 +29,84 @@ function hasScope(file: string) {
   return file.match(/^@/);
 }
 
-export async function findPackages(storagePath: string, validationHandler: Function) {
-  const listPackages: Array<any> = [];
+async function getVersions(scopePath: string, packageName: string): Promise<string[]> {
+  const versions: string[] = [];
+  const arr: string[] = await readDirectory(scopePath);
+  arr.forEach( (filePath: string) => {
+    const fileName = path.basename( filePath);
+    // fileName should start with package name and should ends with '.tgz'
+    if (fileName.indexOf(packageName)===0 && fileName.indexOf('.tgz')!==-1) {
+      const v: string = (fileName.split(packageName+'-')[1] || '').replace('.tgz', '');
+      versions.push(v);
+    }
+  });
+  return versions;
+}
+
+
+export async function findPackages(storagePath: string) {
+  //stats
+  const startTS = Date.now();
+  let packages_count = 0;
+  let versions_count = 0;
+
+  const listPackages: any = {};
   return new Promise(async (resolve, reject) => {
     try {
       const scopePath = path.resolve(storagePath);
       const storageDirs = await readDirectory(scopePath);
       for (const directory of storageDirs) {
-        // we check whether has 2nd level
-        if (hasScope(directory)) {
-          // we read directory multiple
-          const scopeDirectory = path.resolve(storagePath, directory);
-          const scopedPackages = await readDirectory(scopeDirectory);
-          for (const scopedDirName of scopedPackages) {
-            if (validationHandler(scopedDirName)) {
+        const stats = await getFileStats(path.resolve(storagePath, directory));
+        if (stats.isDirectory()) {
+          // we check whether has 2nd level
+          if (hasScope(directory)) {
+            // we read directory multiple
+            const scopeDirectory = path.resolve(storagePath, directory);
+            const scopedPackages = await readDirectory(scopeDirectory);
+            for (const scopedDirName of scopedPackages) {
               // we build the complete scope path
               const scopePath = path.resolve(storagePath, directory, scopedDirName);
-              // list content of such directory
-              listPackages.push({
-                name: `${directory}/${scopedDirName}`,
-                path: scopePath
-              });
+              const stats = await getFileStats(scopePath);
+              if (stats.isDirectory()) {                
+                const versions = await getVersions(scopePath, scopedDirName);
+                // list content of such directory
+                listPackages[`${directory}/${scopedDirName}`] = //{
+//                  path: scopePath,
+                  versions
+//                };
+                packages_count++;
+                versions_count += versions.length;
+              }
+            }
+          } else {
+            // otherwise we read as single level
+            const scopePath = path.resolve(storagePath, directory);  
+            const stats = await getFileStats(scopePath);
+            if (stats.isDirectory()) {                
+
+              const versions = await getVersions(scopePath, directory);
+              listPackages[directory] = //{
+//                path: scopePath,
+                versions
+//              };
+              packages_count++;
+              versions_count += versions.length;
             }
-          }
-        } else {
-          // otherwise we read as single level
-          if (validationHandler(directory)) {
-            const scopePath = path.resolve(storagePath, directory);
-            listPackages.push({
-              name: directory,
-              path: scopePath
-            });
           }
         }
       }
     } catch (error) {
       reject(error);
     }
 
-    resolve(listPackages);
+    resolve({
+      packages: listPackages,
+      stats: {
+        ts: Date.now(),
+        duration: Date.now() - startTS,
+        packages_count,
+        versions_count,
+      }
+    });
   });
-}
+}

tests/utils.test.ts

@@ -50,19 +50,28 @@ describe('Utitlies', () => {
   describe('find packages', () => {
     test('should fails on wrong storage path', async () => {
       try {
-        await findPackages('./no_such_folder_fake', jest.fn(() => true));
+        await findPackages('./no_such_folder_fake');
       } catch (e) {
         expect(e.code).toEqual(noSuchFile);
       }
     });
 
     test('should fetch all packages from valid storage', async () => {
       const storage = path.join(__dirname, '__fixtures__/findPackages');
-      const validator = jest.fn(file => file.indexOf('.') === -1);
-      const pkgs = await findPackages(storage, validator);
-      // the result is 7 due number of packages on "findPackages" directory
-      expect(pkgs).toHaveLength(5);
-      expect(validator).toHaveBeenCalledTimes(8);
+      const pkgs = await findPackages(storage);
+
+      expect(Object.keys(pkgs.packages)).toHaveLength(5);
+      expect(pkgs.stats.packages_count).toBe(5);
+      expect(pkgs.stats.versions_count).toBe(2);
+
+      expect(pkgs.packages['@scoped-test/pkg-1']).toHaveLength(1);
+      expect(pkgs.packages['@scoped-test/pkg2']).toHaveLength(0);
+      expect(pkgs.packages['@scoped_second/pkg1']).toHaveLength(0);
+      expect(pkgs.packages['@scoped_second/pkg2']).toHaveLength(0);
+      expect(pkgs.packages['pk3']).toHaveLength(1);
+
+      expect(pkgs.packages['@scoped-test/pkg-1']).toEqual(['0.1.1-beta.1']);
+      expect(pkgs.packages['pk3']).toEqual(['1.0.0']);
     });
   });
 });

tsconfig.json

@@ -11,7 +11,8 @@
     "typeRoots": [
       "./node_modules/@verdaccio/types/lib/verdaccio",
       "./node_modules/@types"
-    ]
+    ],
+    "experimentalDecorators": true
   },
   "include": [
     "src/*.ts",