Skip to content

Commit 1bf508d

Browse files
dcrescimdcrescim
committed
feat: updated serialize / deserialize to avoid tfjs error
2 parents 76509d8 + b1f646f commit 1bf508d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+762
-695
lines changed

‎docs/convert.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,6 @@ function getTypeName(val, bigObj) {
189189
}
190190

191191
function generateProperties(jsonClass, bigObj) {
192-
// console.log(jsonClass.children)
193192
let interface = getInterfaceForClass(jsonClass, bigObj)
194193
let allConstructorArgs = []
195194
if (interface && interface.children) {

‎package-lock.json‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
"dependencies": {
5353
"@tensorflow/tfjs": "^3.16.0",
5454
"@tensorflow/tfjs-node": "^3.16.0",
55+
"base64-arraybuffer": "^1.0.2",
5556
"lodash": "^4.17.21",
5657
"mathjs": "^10.0.0",
5758
"simple-statistics": "^7.7.0"

‎src/cluster/KMeans.test.ts‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import * as tf from '@tensorflow/tfjs-node'
2-
import { KMeans, setBackend } from '../index'
1+
import * as tf from '@tensorflow/tfjs'
2+
import { KMeans, setBackend, fromJSON } from '../index'
33
setBackend(tf)
44
// Next steps: Improve on kmeans cluster testing
55
describe('KMeans', () => {
@@ -39,7 +39,7 @@ describe('KMeans', () => {
3939
)
4040
})
4141

42-
it('should save kmeans model', () => {
42+
it('should save kmeans model', async () => {
4343
const expectedResult = {
4444
name: 'KMeans',
4545
nClusters: 2,
@@ -49,7 +49,7 @@ describe('KMeans', () => {
4949
randomState: 0,
5050
nInit: 10,
5151
clusterCenters: {
52-
type: 'Tensor',
52+
name: 'Tensor',
5353
value: [
5454
[2.5, 1],
5555
[2.5, 4]
@@ -58,20 +58,21 @@ describe('KMeans', () => {
5858
}
5959
const kmean = new KMeans({ nClusters: 2, randomState: 0 })
6060
kmean.fit(X)
61-
const ksave = kmean.toJson() as string
61+
delete kmean.tf
62+
const ksave = await kmean.toObject()
6263

63-
expect(expectedResult).toEqual(JSON.parse(ksave))
64+
expect(expectedResult).toEqual(ksave)
6465
})
6566

66-
it('should load serialized kmeans model', () => {
67+
it('should load serialized kmeans model', async () => {
6768
const centroids = [
6869
[2.5, 1],
6970
[2.5, 4]
7071
]
7172
const kmean = new KMeans({ nClusters: 2, randomState: 0 })
7273
kmean.fit(X)
73-
const ksave = kmean.toJson() as string
74-
const ksaveModel = new KMeans().fromJson(ksave)
74+
const ksave = await kmean.toJSON()
75+
const ksaveModel = await fromJSON(ksave)
7576
expect(centroids).toEqual(ksaveModel.clusterCenters.arraySync())
7677
})
7778

‎src/cluster/KMeans.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Scikit2D, Tensor1D, Tensor2D } from '../types'
22
import { convertToNumericTensor2D, sampleWithoutReplacement } from '../utils'
3-
import Serialize from '../serialize'
43
import { getBackend } from '../tf-singleton'
4+
import { Serialize } from '../simpleSerializer'
55

66
/*
77
Next steps

‎src/compose/ColumnTransformer.test.ts‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ import {
22
ColumnTransformer,
33
MinMaxScaler,
44
SimpleImputer,
5-
setBackend
5+
setBackend,
6+
fromJSON
67
} from '../index'
78
import * as dfd from 'danfojs-node'
8-
import * as tf from '@tensorflow/tfjs-node'
9+
import * as tf from '@tensorflow/tfjs'
910
setBackend(tf)
1011

1112
describe('ColumnTransformer', function () {
@@ -35,4 +36,26 @@ describe('ColumnTransformer', function () {
3536

3637
expect(result.arraySync()).toEqual(expected)
3738
})
39+
it('ColumnTransformer serialize/deserialize test', async function () {
40+
const X = [
41+
[2, 2], // [1, .5]
42+
[2, 3], // [1, .75]
43+
[0, NaN], // [0, 1]
44+
[2, 0] // [.5, 0]
45+
]
46+
let newDf = new dfd.DataFrame(X)
47+
48+
const transformer = new ColumnTransformer({
49+
transformers: [
50+
['minmax', new MinMaxScaler(), [0]],
51+
['simpleImpute', new SimpleImputer({ strategy: 'median' }), [1]]
52+
]
53+
})
54+
55+
transformer.fitTransform(newDf)
56+
let obj = await transformer.toJSON()
57+
let myResult = await fromJSON(obj)
58+
59+
expect(myResult.transformers.length).toEqual(2)
60+
})
3861
})

‎src/compose/ColumnTransformer.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { DataFrameInterface, Scikit1D, Transformer, Tensor2D } from '../types'
22
import { isDataFrameInterface } from '../typesUtils'
33
import { getBackend } from '../tf-singleton'
4-
4+
import { Serialize } from '../simpleSerializer'
55
/*
66
Next steps:
77
1. Support 'passthrough' and 'drop' and estimator for remainder (also in transformer list)
@@ -65,7 +65,7 @@ export interface ColumnTransformerParams {
6565
]
6666
* ```
6767
*/
68-
export class ColumnTransformer {
68+
export class ColumnTransformer extends Serialize {
6969
transformers: TransformerTriple
7070
remainder: Transformer | 'drop' | 'passthrough'
7171

@@ -77,6 +77,7 @@ export class ColumnTransformer {
7777
transformers = [],
7878
remainder = 'drop'
7979
}: ColumnTransformerParams = {}) {
80+
super()
8081
this.tf = getBackend()
8182
this.transformers = transformers
8283
this.remainder = remainder

‎src/datasets/makeRegression.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { makeLowRankMatrix, makeRegression, setBackend } from '../index'
2-
import * as tf from '@tensorflow/tfjs-node'
2+
import * as tf from '@tensorflow/tfjs'
33
setBackend(tf)
44

55
describe('makeRegression tests', () => {

‎src/dummy/DummyClassifier.test.ts‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { DummyClassifier, setBackend } from '../index'
2-
import * as tf from '@tensorflow/tfjs-node'
1+
import { DummyClassifier, setBackend, fromJSON } from '../index'
2+
import * as tf from '@tensorflow/tfjs'
33
setBackend(tf)
44

55
describe('DummyClassifier', function () {
@@ -53,7 +53,7 @@ describe('DummyClassifier', function () {
5353

5454
expect(scaler.classes).toEqual([1, 2, 3])
5555
})
56-
it('should serialize DummyClassifier', function () {
56+
it('should serialize DummyClassifier', async function () {
5757
const clf = new DummyClassifier()
5858

5959
const X = [
@@ -72,10 +72,12 @@ describe('DummyClassifier', function () {
7272
}
7373

7474
clf.fit(X, y)
75-
const clfSave = clf.toJson() as string
76-
expect(expectedResult).toEqual(JSON.parse(clfSave))
75+
const clfSave = await clf.toObject()
76+
// We don't care what version of tf is saved on there
77+
delete clfSave.tf
78+
expect(expectedResult).toEqual(clfSave)
7779
})
78-
it('should load DummyClassifier', function () {
80+
it('should load DummyClassifier', async function () {
7981
const clf = new DummyClassifier()
8082

8183
const X = [
@@ -87,8 +89,8 @@ describe('DummyClassifier', function () {
8789
const y = [10, 20, 20, 30]
8890

8991
clf.fit(X, y)
90-
const clfSave = clf.toJson() as string
91-
const newClf = new DummyClassifier().fromJson(clfSave)
92+
const clfSave = await clf.toJSON()
93+
const newClf = await fromJSON(clfSave)
9294
expect(clf).toEqual(newClf)
9395
})
9496
})

‎src/dummy/DummyRegressor.test.ts‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { DummyRegressor, setBackend } from '../index'
2-
import * as tf from '@tensorflow/tfjs-node'
1+
import { DummyRegressor, setBackend, fromJSON } from '../index'
2+
import * as tf from '@tensorflow/tfjs'
33
setBackend(tf)
44

55
describe('DummyRegressor', function () {
@@ -57,7 +57,7 @@ describe('DummyRegressor', function () {
5757
reg.fit(X, y)
5858
expect(reg.predict(predictX).arraySync()).toEqual([10, 10, 10])
5959
})
60-
it('Should save DummyRegressor', function () {
60+
it('Should save DummyRegressor', async function () {
6161
const reg = new DummyRegressor({ strategy: 'constant', constant: 10 })
6262

6363
const X = [
@@ -70,15 +70,16 @@ describe('DummyRegressor', function () {
7070
name: 'DummyRegressor',
7171
EstimatorType: 'regressor',
7272
strategy: 'constant',
73-
constant: 10
73+
constant: 10,
74+
quantile: undefined
7475
}
7576

7677
reg.fit(X, y)
77-
78-
expect(saveResult).toEqual(JSON.parse(reg.toJson() as string))
78+
delete reg.tf
79+
expect(saveResult).toEqual(await reg.toObject())
7980
})
8081

81-
it('Should load serialized DummyRegressor', function () {
82+
it('Should load serialized DummyRegressor', async function () {
8283
const reg = new DummyRegressor({ strategy: 'constant', constant: 10 })
8384

8485
const X = [
@@ -94,8 +95,8 @@ describe('DummyRegressor', function () {
9495
]
9596

9697
reg.fit(X, y)
97-
const saveReg = reg.toJson() as string
98-
const newReg = new DummyRegressor().fromJson(saveReg)
98+
const saveReg = await reg.toJSON()
99+
const newReg = await fromJSON(saveReg)
99100

100101
expect(newReg.predict(predictX).arraySync()).toEqual([10, 10, 10])
101102
})

0 commit comments

Comments
 (0)