initial atari tests (not with image) and dither improvements

palette selection now biases to matches on the bright side of
the RGB color cube, since black is always available for dithering

does better but still not perfect

atari binary just writes some test data to text framebuffer
but builds and runs

Makefile

@@ -0,0 +1,19 @@
+.PHONY : all clean
+
+all : dither4.xex
+
+# sample5.s from sample5.jpg
+
+# reminder: $< is input
+#           $@ is output
+%.o : %.s
+	ca65 -v -t atari -o $@ $<
+
+dither4.xex : dither4.o
+	ld65 -v -C atari-asm-xex.cfg -o $@ $< 
+
+clean :
+	rm -f sample5.s
+	rm -f sample5.o
+	rm -f dither4.o
+	rm -f dither4.xex

dither4.js

@@ -388,15 +388,19 @@ function decimate(input, palette, n) {
         for (let x = 0; x < line.length; x++) {
             let rgb = line[x];
             rgb = rgb.add(error.right);
-            rgb.cap();
+            //rgb.cap();
 
             // find the closest possible color
             let shortest = Infinity;
-            let pick = -1;
+            let pick = 1;
 
             for (let i = 0; i < palette.length; i++) {
                 let diff = rgb.difference(palette[i]);
                 let dist = diff.magnitude();
+                let darker = Math.min(diff.r, diff.g, diff.b) < 0;
+                if (darker) {
+                    dist **= 2;
+                }
                 if (dist < shortest) {
                     nextError = diff;
                     shortest = dist;
@@ -430,7 +434,6 @@ function decimate(input, palette, n) {
             error.blue[x] += nextError.b / 2;
             */
 
-            /*
             error.right.r = nextError.r / 2;
             error.right.g = nextError.g / 2;
             error.right.b = nextError.b / 2;
@@ -446,8 +449,8 @@ function decimate(input, palette, n) {
             error.red[x + 1] += nextError.r / 8;
             error.green[x + 1] += nextError.g / 8;
             error.blue[x + 1] += nextError.b / 8;
-            */
 
+            /*
             error.right.r = nextError.r / 4;
             error.right.g = nextError.g / 4;
             error.right.b = nextError.b / 4;
@@ -463,9 +466,20 @@ function decimate(input, palette, n) {
             error.red[x + 1] += nextError.r / 4;
             error.green[x + 1] += nextError.g / 4;
             error.blue[x + 1] += nextError.b / 4;
+            */
 
             // 442 is the 3d distance across the rgb cube
-            fitness[x] = 442 - (nextError.magnitude());
+            //fitness[x] = 442 - (nextError.magnitude());
+            //fitness[x] = 442 / (442 - nextError.magnitude());
+            fitness[x] = 255 / (256 - Math.max(nextError.r, nextError.g, nextError.b));
+            
+            /*
+            fitness[x] = Math.max(
+                255 - Math.abs(nextError.r),
+                255 - Math.abs(nextError.g),
+                255 - Math.abs(nextError.b),
+            );
+            */
         }
         return {
             output,
@@ -476,8 +490,37 @@ function decimate(input, palette, n) {
         };
     };
 
+
+    // black, red, blue, white
+    let rbw = [
+        palette256[0x00],
+        palette256[0x87],
+        palette256[0xf7],
+        palette256[0x0f],
+    ];
+
+    let rgb = [
+        palette256[0x00],
+        palette256[0x87],
+        palette256[0xc7],
+        palette256[0xf7],
+    ];
+
+    // grayscale
+    let gray = [
+        palette256[0x00],
+        palette256[0x05],
+        palette256[0x0a],
+        palette256[0x0f],
+    ];
+
+    //palette = rgb;
+    //palette = rbw;
+    //palette = gray;
+
     let start = Date.now();
     let decimated = palette.slice();
+
     while (decimated.length > n) {
         let {popularity, fitness, output} = dither(decimated);
 
@@ -489,21 +532,17 @@ function decimate(input, palette, n) {
                 continue; // keep black always
             }
             if (decimated[i].r == 255 && decimated[i].g == 255 && decimated[i].b == 255) {
-                continue; // keep white always
+                //continue; // keep white always
             }
 
-            let coolFactor = popularity[i];
+            //let coolFactor = popularity[i];
-
-            /*
 
             let coolFactor = 0;
             for (let x = 0; x < line.length; x++) {
                 if (output[x] == i) {
-                    coolFactor += fitness[x];
+                    coolFactor += fitness[x] ** 3;
                 }
             }
-            coolFactor /= Math.sqrt(popularity[i]);
-            */
 
             if (coolFactor < least) {
                 pick = i;

dither4.s

@@ -0,0 +1,23 @@
+SAVMSC = $58
+
+.code
+
+.export start
+
+.proc start
+    ; Get the framebuffer address and just write new fun values into it
+    ; x: byte to write in
+    ; y: index into the first 256 bytes of buffer
+
+    ldx #00
+outer_loop:
+    txa
+    ldy #00
+inner_loop:
+    sta (SAVMSC),y
+    iny
+    bne inner_loop
+    inx
+    jmp outer_loop
+    ; infinite loop
+.endproc