Draft fix for utf-8 issues.

content/src/filetype.js

@@ -9,6 +9,11 @@ function inferScriptType(filename) {
   return mime.replace(/;.*$/, '');
 }
 
+function binaryTypeFilename(filename) {
+  var mime = mimeForFilename(filename);
+  return !(/^text\//.test(mime));
+}
+
 // Scans for HTML HEAD content at the top, remembering the positions
 // after any start-tags seen and before any legal end-tags.
 // Returns {
@@ -319,6 +324,7 @@ function isDefaultMeta(meta) {
 
 var impl = {
   mimeForFilename: mimeForFilename,
+  binaryTypeFilename: binaryTypeFilename,
   modifyForPreview: modifyForPreview,
   effectiveMeta: effectiveMeta,
   isDefaultMeta: isDefaultMeta,

server/filemeta.js

@@ -21,7 +21,7 @@ var delimiter = [
   }
 ];
 
-function parseMetaString(buf) {
+function parseMetaString(buf, assumeBinary) {
   var j, d, limit, start, end, meta, enc, str = buf.toString('binary');
   for (j = 0; j < delimiter.length; ++j) {
     d = delimiter[j];
@@ -39,18 +39,30 @@ function parseMetaString(buf) {
       if (meta && meta.encoding && Buffer.isEncoding(meta.encoding)) {
         enc = meta.encoding;
       } else {
-        enc = 'binary';
+        enc = assumeBinary ? 'binary' : 'utf8';
+      }
+      var decoded = buf.toString(enc, 0, limit);
+      // Deal with legacy data that might have been encoded as binary
+      // when the new default for text files is utf8.
+      if (enc == 'utf8' && decoded.indexOf('\ufffd') != -1) {
+        var recoded = new Buffer(decoded, enc);
+        // The utf8 is malformed if reencoding it results in different bytes.
+        if (recoded.length != limit || !recoded.equals(buf.slice(0, limit))) {
+          // In this case, treat the encoding as binary instead.
+          enc = 'binary';
+          decoded = buf.toString(enc, 0, limit);
+        }
       }
       return {
-        data: buf.toString(enc, 0, limit),
+        data: decoded,
         meta: meta
       }
     } catch (e) { }
   }
   return { data: str, meta: null };
 }
 
-function printMetaString(data, meta) {
+function printMetaString(data, meta, assumeBinary) {
   if (meta == null &&
       (data.lastIndexOf('META@') == -1 || data.lastIndexOf('@META') == -1) &&
       /^[\0-\xff]*$/.test(data)) {
@@ -62,7 +74,7 @@ function printMetaString(data, meta) {
       if (delimiter[j].type.test(meta.type)) { d = delimiter[j]; break; }
     }
   }
-  var enc = 'binary';
+  var enc = assumeBinary ? 'binary' : 'utf8';
   if (meta && meta.encoding && Buffer.isEncoding(meta.encoding)) {
     enc = meta.encoding;
   } else if (!/^[\0-\xff]*$/.test(data)) {

server/load.js

@@ -96,6 +96,7 @@ exports.handleLoad = function(req, res, app, format) {
         utils.errorExit('Bad filename: ' + filename);
       }
     }
+    var assumeBinary = filetype.binaryTypeFilename(filename);
 
     var absfile = utils.makeAbsolute(filename, app);
 
@@ -105,7 +106,7 @@ exports.handleLoad = function(req, res, app, format) {
       // Handle the case of a file that's present
       if (utils.isPresent(absfile, 'file')) {
         var m = filemeta.parseMetaString(
-            fs.readFileSync(absfile)),
+                fs.readFileSync(absfile), assumeBinary),
             data = m.data,
             meta = m.meta;
 
@@ -161,7 +162,7 @@ exports.handleLoad = function(req, res, app, format) {
     else if (format == 'code') { // For loading the code only
       var mt = filetype.mimeForFilename(filename),
           m = filemeta.parseMetaString(
-              fs.readFileSync(absfile)),
+              fs.readFileSync(absfile), assumeBinary),
           data = m.data,
           meta = m.meta;
       res.set('Cache-Control', 'must-revalidate');
@@ -173,7 +174,7 @@ exports.handleLoad = function(req, res, app, format) {
     else if (format == 'print') { // For printing the code
       var mt = filetype.mimeForFilename(filename),
           m = filemeta.parseMetaString(
-              fs.readFileSync(absfile)),
+              fs.readFileSync(absfile), assumeBinary),
           data = m.data,
           meta = m.meta,
           needline = false,
@@ -224,7 +225,7 @@ exports.handleLoad = function(req, res, app, format) {
       if (utils.isPresent(absfile, 'file')) {
         var mt = filetype.mimeForFilename(filename),
             m = filemeta.parseMetaString(
-                fs.readFileSync(absfile));
+                fs.readFileSync(absfile), assumeBinary);
 
         // For turtle bits, assume it's coffeescript
         if (mt.indexOf('text/x-pencilcode') == 0) {

server/save.js

@@ -300,7 +300,8 @@ exports.handleSave = function(req, res, app) {
 
     var statObj;
     try {
-      var content = filemeta.printMetaString(data, meta);
+      var assumeBinary = filetype.binaryTypeFilename(filename);
+      var content = filemeta.printMetaString(data, meta, assumeBinary);
       fd = fs.writeFileSync(absfile, content);
       var statObj = fs.statSync(absfile);
       touchUserDir(userdir);