Add transform preprocess

ai-on-browser · Jun 10, 2024 · 5d5992a · 5d5992a
1 parent 630d644
commit 5d5992a
Show file tree

Hide file tree

Showing 12 changed files with 195 additions and 293 deletions.
diff --git a/js/data/dashboard_estat.js b/js/data/dashboard_estat.js
@@ -62,9 +62,6 @@ export default class EStatData extends FixData {
 	constructor(manager) {
 		super(manager)
 		this._name = 'Nikkei Indexes'
-		this._shift = []
-		this._scale = []
-		this._scaled = true
 		this._lastRequested = 0
 
 		const elm = this.setting.data.configElement
@@ -118,16 +115,6 @@ export default class EStatData extends FixData {
 		this._loader = document.createElement('div')
 		elm.appendChild(this._loader)
 
-		const optionalElm = document.createElement('div')
-		const scaledCheckbox = document.createElement('input')
-		scaledCheckbox.type = 'checkbox'
-		scaledCheckbox.checked = true
-		scaledCheckbox.onchange = () => {
-			this._scaled = scaledCheckbox.checked
-			this._readyData()
-		}
-		optionalElm.append('Scale', scaledCheckbox)
-		elm.appendChild(optionalElm)
 		this._initIndicatorSelector().then(() => {
 			const info = presetInfos[this._name]
 			if (info) {
@@ -160,15 +147,7 @@ export default class EStatData extends FixData {
 	}
 
 	get x() {
-		if (!this._scaled) return this.originalX
-		if (this._requireDateInput) {
-			return this._datetime.map(v => [v])
-		}
-		this._readyScaledData()
-		return this._x.map(v => {
-			const c = v.map((a, d) => (a - this._shift[d]) / this._scale[d])
-			return this._selector.object.map(i => c[i])
-		})
+		return this.originalX
 	}
 
 	get originalY() {
@@ -180,13 +159,7 @@ export default class EStatData extends FixData {
 	}
 
 	get y() {
-		if (!this._scaled) return this.originalY
-		this._readyScaledData()
-		const target = this._selector.target
-		if (target >= 0) {
-			return this._x.map(v => (v[target] - this._shift[target]) / this._scale[target])
-		}
-		return Array(this._x.length).fill(0)
+		return this.originalY
 	}
 
 	get params() {
@@ -404,33 +377,6 @@ export default class EStatData extends FixData {
 		}
 	}
 
-	_readyScaledData() {
-		if (this._scale.length > 0) {
-			return
-		}
-		this._shift = []
-		this._scale = []
-		if (this._x.length > 0) {
-			const min = Array(this._x[0].length).fill(Infinity)
-			const max = Array(this._x[0].length).fill(-Infinity)
-			for (let i = 0; i < this._x.length; i++) {
-				for (let d = 0; d < this._x[i].length; d++) {
-					min[d] = Math.min(min[d], this._x[i][d])
-					max[d] = Math.max(max[d], this._x[i][d])
-				}
-			}
-			const rmax = 10
-			const rmin = 0
-			for (let d = 0; d < min.length; d++) {
-				if (min[d] === max[d]) {
-					max[d] = min[d] + 1
-				}
-				this._scale[d] = (max[d] - min[d]) / (rmax - rmin)
-				this._shift[d] = min[d] - rmin * this._scale[d]
-			}
-		}
-	}
-
 	async _getMeta(func) {
 		const keySuffix = func.toUpperCase() === 'INDICATOR' || func.toUpperCase() === 'REGION' ? 'INF' : 'INFO'
 		const metaKey = `GET_META_${func.toUpperCase()}_${keySuffix}`
@@ -530,8 +476,6 @@ export default class EStatData extends FixData {
 
 	async _readyData() {
 		this._x = []
-		this._shift = []
-		this._scale = []
 		this._index = null
 		this._datetime = null
 		this._manager.platform?.init()

diff --git a/js/data/eurostat.js b/js/data/eurostat.js
@@ -30,9 +30,6 @@ export default class EurostatData extends FixData {
 	constructor(manager) {
 		super(manager)
 		this._name = 'Population and employment'
-		this._shift = []
-		this._scale = []
-		this._scaled = true
 		this._filterItems = null
 		this._lastRequested = 0
 
@@ -88,17 +85,6 @@ export default class EurostatData extends FixData {
 		this._loader = document.createElement('div')
 		elm.appendChild(this._loader)
 
-		const optionalElm = document.createElement('div')
-		const scaledCheckbox = document.createElement('input')
-		scaledCheckbox.type = 'checkbox'
-		scaledCheckbox.checked = true
-		scaledCheckbox.onchange = () => {
-			this._scaled = scaledCheckbox.checked
-			this._readyData()
-		}
-		optionalElm.append('Scale', scaledCheckbox)
-		elm.appendChild(optionalElm)
-
 		this._readyData()
 	}
 
@@ -125,15 +111,7 @@ export default class EurostatData extends FixData {
 	}
 
 	get x() {
-		if (!this._scaled) return this.originalX
-		if (this._requireDateInput) {
-			return this._datetime.map(v => [v])
-		}
-		this._readyScaledData()
-		return this._x.map(v => {
-			const c = v.map((a, d) => (a - this._shift[d]) / this._scale[d])
-			return this._selector.object.map(i => c[i])
-		})
+		return this.originalX
 	}
 
 	get originalY() {
@@ -144,13 +122,7 @@ export default class EurostatData extends FixData {
 	}
 
 	get y() {
-		if (!this._scaled) return this.originalY
-		this._readyScaledData()
-		const target = this._selector.target
-		if (target >= 0) {
-			return this._x.map(v => (v[target] - this._shift[target]) / this._scale[target])
-		}
-		return Array(this._x.length).fill(0)
+		return this.originalY
 	}
 
 	get params() {
@@ -166,33 +138,6 @@ export default class EurostatData extends FixData {
 		}
 	}
 
-	_readyScaledData() {
-		if (this._scale.length > 0) {
-			return
-		}
-		this._shift = []
-		this._scale = []
-		if (this._x.length > 0) {
-			const min = Array(this._x[0].length).fill(Infinity)
-			const max = Array(this._x[0].length).fill(-Infinity)
-			for (let i = 0; i < this._x.length; i++) {
-				for (let d = 0; d < this._x[i].length; d++) {
-					min[d] = Math.min(min[d], this._x[i][d])
-					max[d] = Math.max(max[d], this._x[i][d])
-				}
-			}
-			const rmax = 10
-			const rmin = 0
-			for (let d = 0; d < min.length; d++) {
-				if (min[d] === max[d]) {
-					max[d] = min[d] + 1
-				}
-				this._scale[d] = (max[d] - min[d]) / (rmax - rmin)
-				this._shift[d] = min[d] - rmin * this._scale[d]
-			}
-		}
-	}
-
 	async _getData(datasetCode, query) {
 		const params = {
 			format: 'JSON',
@@ -239,8 +184,6 @@ export default class EurostatData extends FixData {
 
 	async _readyData() {
 		this._x = []
-		this._shift = []
-		this._scale = []
 		this._index = null
 		this._datetime = null
 		this._manager.platform?.init()

diff --git a/js/model_selector.js b/js/model_selector.js
@@ -62,7 +62,11 @@ const AITask = {
 const AIPreprocess = {
 	function: {
 		title: 'Basis function',
-		tasks: ['CF', 'RG', 'RL'],
+		tasks: ['CF', 'SC', 'RG', 'IN', 'RL', 'AD', 'DR', 'CP'],
+	},
+	transform: {
+		title: 'Transformers',
+		tasks: ['CT', 'CF', 'SC', 'RG', 'IN', 'RL', 'AD', 'DR', 'FS', 'SM', 'TP', 'CP'],
 	},
 }
 for (const ap of Object.keys(AIPreprocess)) {

diff --git a/js/platform/base.js b/js/platform/base.js
@@ -149,30 +149,6 @@ export class DefaultPlatform extends BasePlatform {
 		this._renderer[0].testResult(pred)
 	}
 
-	evaluate(cb) {
-		if (this.task !== 'CF' && this.task !== 'RG' && this.task !== 'RL') {
-			return
-		}
-		cb(this.datas.x, p => {
-			const t = this.datas.y
-			if (this.task === 'CF' || this.task === 'RL') {
-				let acc = 0
-				for (let i = 0; i < t.length; i++) {
-					if (t[i] === p[i]) {
-						acc++
-					}
-				}
-				this._getEvaluateElm().innerText = 'Accuracy:' + acc / t.length
-			} else if (this.task === 'RG') {
-				let rmse = 0
-				for (let i = 0; i < t.length; i++) {
-					rmse += (t[i] - p[i]) ** 2
-				}
-				this._getEvaluateElm().innerText = 'RMSE:' + Math.sqrt(rmse / t.length)
-			}
-		})
-	}
-
 	init() {
 		this._cur_dimension = this.setting.dimension
 		this.setting.footer.innerText = ''
@@ -189,10 +165,24 @@ export class DefaultPlatform extends BasePlatform {
 		}
 	}
 
+	invertScale(x) {
+		for (const preprocess of this._manager.preprocesses) {
+			if (preprocess.inverse) {
+				if (Array.isArray(x[0])) {
+					x = preprocess.inverse(x)
+				} else {
+					x = preprocess.inverse([x])[0]
+				}
+			}
+		}
+		return x
+	}
+
 	centroids(center, cls, { line = false, duration = 0 } = {}) {
 		if (!this._centroids) {
 			this._centroids = new CentroidPlotter(this._renderer[0])
 		}
+		center = this.invertScale(center)
 		this._centroids.set(center, cls, { line, duration })
 	}
 

diff --git a/js/platform/series.js b/js/platform/series.js
@@ -61,6 +61,19 @@ export default class SeriesPlatform extends BasePlatform {
 		}
 	}
 
+	invertScale(x) {
+		for (const preprocess of this._manager.preprocesses) {
+			if (preprocess.inverse) {
+				if (Array.isArray(x[0])) {
+					x = preprocess.inverse(x)
+				} else {
+					x = preprocess.inverse([x])[0]
+				}
+			}
+		}
+		return x
+	}
+
 	resetPredicts() {
 		this._renderer.forEach(rend => rend.resetPredicts())
 	}

diff --git a/js/preprocess/transform.js b/js/preprocess/transform.js
@@ -0,0 +1,63 @@
+import MinmaxNormalization from '../../lib/model/minmax.js'
+import Standardization from '../../lib/model/standardization.js'
+import MaxAbsScaler from '../../lib/model/maxabs.js'
+import RobustScaler from '../../lib/model/robust_scaler.js'
+import BoxCox from '../../lib/model/box_cox.js'
+import YeoJohnson from '../../lib/model/yeo_johnson.js'
+
+const transformers = {
+	minmax: MinmaxNormalization,
+	standard: Standardization,
+	maxabs: MaxAbsScaler,
+	robust: RobustScaler,
+	'Box-Cox': BoxCox,
+	'Yeo-Johnson': YeoJohnson,
+}
+
+export default class TransformPreprocessor {
+	constructor(manager) {
+		this._manager = manager
+		this._method = 'standard'
+
+		this.init()
+	}
+
+	init() {
+		if (!this._r) {
+			const elm = this._manager.setting.preprocess.configElement
+			this._r = document.createElement('div')
+			elm.append(this._r)
+		} else {
+			this._r.replaceChildren()
+		}
+		const methodElm = document.createElement('div')
+		const method = document.createElement('select')
+		for (const key of Object.keys(transformers)) {
+			const opt = method.appendChild(document.createElement('option'))
+			opt.value = opt.innerText = key
+		}
+		method.onchange = () => {
+			this._method = method.value
+			this._manager.setting.ml.refresh()
+		}
+		method.value = this._method
+		methodElm.append('Method ', method)
+		this._r.append(methodElm)
+	}
+
+	apply(x, { dofit = true }) {
+		if (dofit) {
+			this._model = new transformers[this._method]()
+			this._model.fit(x)
+		}
+		return this._model.predict(x)
+	}
+
+	inverse(z) {
+		return this._model.inverse(z)
+	}
+
+	terminate() {
+		this._r?.remove()
+	}
+}
diff --git a/js/renderer/line.js b/js/renderer/line.js
@@ -422,10 +422,15 @@ export default class LineRenderer extends BaseRenderer {
 			const datas = this.datas
 			const path = []
 			if (datas.length > 0) {
-				path.push(this.toPoint([datas.length - 1, datas.x[datas.length - 1] || [datas.y[datas.length - 1]]]))
+				path.push(
+					this.toPoint([
+						datas.length - 1,
+						datas.dimension > 0 ? datas.x[datas.length - 1] : [datas.y[datas.length - 1]],
+					])
+				)
 			}
 			for (let i = 0; i < pred.length; i++) {
-				const a = this.toPoint([i + datas.length, pred[i]])
+				const a = this.toPoint([i + datas.length, this._manager.platform.invertScale(pred[i])])
 				const p = new DataPoint(this._r_tile, a, specialCategory.dummy)
 				path.push(a)
 				this._pred_points.push(p)
@@ -439,7 +444,7 @@ export default class LineRenderer extends BaseRenderer {
 		} else if (task === 'SM') {
 			const path = []
 			for (let i = 0; i < pred.length; i++) {
-				const a = this.toPoint([i, pred[i]])
+				const a = this.toPoint([i, this._manager.platform.invertScale(pred[i])])
 				path.push(a)
 			}
 			if (path.length > 0) {