Merge pull request #72 from JasonShin/feature/datasets-boston

JasonShin · JasonShin · commit e5b2bfe57cfa · 2018-09-05T14:59:13.000+10:00
diff --git a/docs/processor/ExampleProcessor.ts b/docs/processor/ExampleProcessor.ts
@@ -41,7 +41,7 @@ export class ExampleProcessor extends BaseProcesser {
       };
     });
     const extraConfig = {
-      exampleSidebar: config,
+      exampleSidebar: config
     };
     // Writing extraConfig object as .vuepress/exampleExtra.json
     fs.writeFileSync(this.vuepressExampleConfigPath, JSON.stringify(extraConfig), 'utf-8');
diff --git a/package.json b/package.json
@@ -35,10 +35,11 @@
     "OneHotEncoding"
   ],
   "scripts": {
-    "build": "yarn clean && yarn build:main && yarn build:module && yarn build:web",
+    "build": "yarn clean && yarn build:main && yarn build:module && yarn build:web && yarn build:datasets",
     "build:main": "npx tsc -p tsconfig.json && npx copy package.json ./build/lib && npx copy README.md ./build/lib && npx copy .npmignore ./build/lib",
     "build:module": "npx tsc -p tsconfig.module.json",
     "build:web": "npx tsc -p tsconfig.json --module amd --out build/lib/kalimdor.web.js",
+    "build:datasets": "cp -r src/lib/datasets/data build/lib/datasets/data",
     "fix": "yarn fix:prettier && yarn fix:tslint",
     "fix:prettier": "npx prettier \"src/**/*.ts\" \"test/**/*.ts\" \"docs/**/*.ts\" --config ./.prettierrc --write",
     "fix:tslint": "npx tslint --fix --force -t verbose 'test/**/*.ts' 'src/**/*.ts' 'docs/**/*.ts'",
diff --git a/src/lib/datasets/BaseDataset.ts b/src/lib/datasets/BaseDataset.ts
@@ -1,5 +1,7 @@
+import * as fs from 'fs-extra';
 import 'isomorphic-fetch';
 import { parseInt, uniqBy } from 'lodash';
+import * as path from 'path';
 import { LabelEncoder } from '../preprocessing/label';
 
 /**
@@ -57,6 +59,24 @@ export class BaseDataset {
     };
   }
 
+  /**
+   * Load data from the local data folder
+   */
+  protected async fsLoad(
+    type: string,
+    { delimiter = ',', lastIsTarget = true, trainType = 'float', targetType = 'float' } = {
+      // Default object if nothing is provided
+      delimiter: ',',
+      lastIsTarget: true,
+      trainType: 'float',
+      targetType: 'float'
+    }
+  ): Promise<{ data; targets; labels }> {
+    // Make sure the actual data is located under data/type
+    const data = fs.readFileSync(path.join(__dirname, `data/${type}/train.csv`), 'utf8');
+    return this.processCSV(data, delimiter, lastIsTarget, trainType, targetType);
+  }
+
   /**
    * Processes CSV type dataset. Returns a training and testing data pair
    * @param data - a raw string data
@@ -119,7 +139,6 @@ export class BaseDataset {
     } else if (targetType === 'float') {
       result[1] = result[1].map(parseFloat);
     }
-
     return {
       data: result[0],
       targets,
diff --git a/src/lib/datasets/Boston.ts b/src/lib/datasets/Boston.ts
@@ -0,0 +1,58 @@
+/**
+ * References:
+ * - https://www.kaggle.com/c/boston-housing/data
+ * - https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html
+ */
+
+import { BaseDataset } from './BaseDataset';
+
+/**
+ * This dataset contains information collected by the U.S Census Service concerning housing in the area of Boston Mass.
+ * It was obtained from the StatLib archive (http://lib.stat.cmu.edu/datasets/boston),
+ * and has been used extensively throughout the literature to benchmark algorithms.
+ * However, these comparisons were primarily done outside of Delve and are thus somewhat suspect.
+ * The dataset is small in size with only 506 cases.
+ *
+ * The data was originally published by Harrison, D. and Rubinfeld, D.L.
+ * `Hedonic prices and the demand for clean air', J. Environ. Economics & Management, vol.5, 81-102, 1978.
+ *
+ * @example
+ * import { Boston } from "kalimdor/datasets";
+ *
+ * (async function() {
+ *   const bostonData = new Boston();
+ *   const {
+ *     data,
+ *     targets,
+ *     labels,
+ *   } = await bostonData.load();
+ * });
+ *
+ */
+export class Boston extends BaseDataset {
+  /**
+   * Load the dataset
+   */
+  public async load(): Promise<{
+    /**
+     * Training data
+     */
+    data: any[][];
+    /**
+     * Target data
+     */
+    targets: any[];
+    /**
+     * Real labels
+     */
+    labels: string[];
+  }> {
+    const { data, targets, labels } = await this.fsLoad('boston');
+    // console.info('checking data', data[0]);
+    return {
+      data,
+      targets,
+      labels
+    };
+  }
+}
diff --git a/src/lib/datasets/Iris.ts b/src/lib/datasets/Iris.ts
@@ -23,7 +23,7 @@ import { BaseDataset } from './BaseDataset';
  *     labels,       // list of labels
  *     targetNames,  // list of short target labels
  *     description   // dataset description
- *   } = irisData.load(); // loads the data internally
+ *   } = await irisData.load(); // loads the data internally
  * })();
  *
  */
diff --git a/src/lib/datasets/data/boston/train.csv b/src/lib/datasets/data/boston/train.csv
diff --git a/src/lib/datasets/index.repl.ts b/src/lib/datasets/index.repl.ts
@@ -6,3 +6,10 @@ irisData.load().then(data => {
   const { description } = data;
   console.log('checking desc', description);
 });
+
+import { Boston } from './Boston';
+
+const bostonData = new Boston();
+bostonData.load().then(data => {
+  console.log(data);
+});
diff --git a/test/datasets/Boston.test.ts b/test/datasets/Boston.test.ts
@@ -0,0 +1,22 @@
+import { Boston } from '../../src/lib/datasets/Boston';
+
+describe('datasets:Boston', () => {
+  let boston = null;
+  beforeAll(() => {
+    boston = new Boston();
+  });
+  it('should data match the snapshot', async () => {
+    const { data } = await boston.load();
+    expect(data).toMatchSnapshot();
+  });
+
+  it('should targets match the snapshot', async () => {
+    const { targets } = await boston.load();
+    expect(targets).toMatchSnapshot();
+  });
+
+  it('should labels match the snapshot', async () => {
+    const { labels } = await boston.load();
+    expect(labels).toMatchSnapshot();
+  });
+});
diff --git a/test/datasets/__snapshots__/Boston.test.ts.snap b/test/datasets/__snapshots__/Boston.test.ts.snap
diff --git a/test/datasets/fake_data/boston.ts b/test/datasets/fake_data/boston.ts

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,7 @@ import { BaseDataset } from './BaseDataset';`
`23`	`23`	`* labels, // list of labels`
`24`	`24`	`* targetNames, // list of short target labels`
`25`	`25`	`* description // dataset description`
`26`		`- * } = irisData.load(); // loads the data internally`
	`26`	`+ * } = await irisData.load(); // loads the data internally`
`27`	`27`	`* })();`
`28`	`28`	`*`
`29`	`29`	`*/`