slightly smaller

scratch.js

@@ -2,7 +2,7 @@ import { learn, reverse, test, compress, uncompress, classify, fingerprint, tiny
 import data from '/Users/spencer/mountain/it-compromise/data/models/verbs/conditional.js'
 
 // let pairs = Object.keys(data).map(k => [k, data[k][0]])
-let pairs = Object.keys(data).map(k => [data[k][0], k]).slice(0, 200)
+let pairs = Object.keys(data).map(k => [data[k][0], k])//.slice(0, 200)
 // pairs = [
 //   ['sottomettere', 'sottometterei'],
 //   ['teletrasmettere', 'teletrasetterei'],
@@ -12,13 +12,11 @@ let pairs = Object.keys(data).map(k => [data[k][0], k]).slice(0, 200)
 
 // console.log(compress(learn(pairs)))
 let res = tiny(pairs)
-// res = compress(res)
 console.log(res)
 console.log('\n\n')
 let small = pack(res)
 console.log(small)
 let again = unpack(small)
-console.log(again)
 test(pairs, again)
 
 // { ei: 'e' }, 

src/pack/key-val.js

@@ -0,0 +1,25 @@
+// longest common prefix
+const findOverlap = (from, to) => {
+  let all = []
+  for (let i = 0; i < from.length; i += 1) {
+    if (from[i] === to[i]) {
+      all.push(from[i])
+    } else {
+      break
+    }
+  }
+  return all.join('')
+}
+
+// run-length encode any shared prefix
+let compress = function (key, val) {
+  let prefix = findOverlap(key, val)
+  if (prefix.length < 1) {
+    return val
+  }
+  let out = prefix.length + val.substr(prefix.length)
+  return out
+}
+
+export default compress
+// console.log(compress('fixture', 'fixturing'))

src/pack/pack.js

@@ -1,18 +1,28 @@
+import keyVal from './key-val.js'
+
 const pack = function (model) {
   let out = ''
   model.rules.forEach(obj => {
     let r = []
     Object.keys(obj).forEach(k => {
-      r.push(k + '|' + obj[k] + ',')
+      let val = keyVal(k, obj[k])// compress any shared prefix
+      if (!val) {
+        console.log('=-=-=-= here -=-=-=-')
+        console.log(k, obj[k])
+      }
+      r.push(k + ':' + val + ',')
     })
     out += r.join('')
   })
   out += '=='
   let r = []
-  Object.keys(model.exceptions).forEach(k => {
-    r.push(k + '|' + model.exceptions[k])
+  Object.keys(model.exceptions || {}).forEach(k => {
+    let val = keyVal(k, model.exceptions[k])// compress any shared prefix
+    r.push(k + ':' + val)
   })
   out += r.join(',')
   return out
 }
-export default pack
+export default pack
+
+// rei:ere, erei:are, rrei:nere, arei:3,  herei:are,  lirei:4,   nirei:4, tirei:4,==

src/pack/unpack.js

@@ -1,13 +1,27 @@
+const prefix = /^([0-9]+)/
+
+const expand = function (key = '', val = '') {
+  val = String(val)
+  let m = val.match(prefix)
+  if (m === null) {
+    return val
+  }
+  let num = Number(m[1]) || 0
+  let pre = key.substring(0, num)
+  let full = pre + val.replace(prefix, '')
+  return full
+}
+
 const unpack = function (str) {
   let out = { rules: [], exceptions: {} }
   let [rules, exceptions] = str.split('==')
   // unpack rules
   rules.split(',').forEach(txt => {
-    let [a, b] = txt.split('|')
+    let [a, b] = txt.split(':')
     let len = a.length
     if (len) {
       out.rules[len] = out.rules[len] || {}
-      out.rules[len][a] = b
+      out.rules[len][a] = expand(a, b)
     }
   })
   // clean empties up a bit
@@ -17,8 +31,8 @@ const unpack = function (str) {
 
   // unpack exceptions
   exceptions.split(',').forEach(txt => {
-    let [a, b] = txt.split('|')
-    out.exceptions[a] = b
+    let [a, b] = txt.split(':')
+    out.exceptions[a] = expand(a, b)
   })
   return out
 }