renderdiff: add viewer for image differences (#8768)

- Modify the compare script to output more details of a
   comparison. This will include the source/golden directory,
   the comparison directory (the new renderings), and a file
   path to difference images if the golden does not match
   the rendered image.
 - The image_diff script can now output a TIFF that is the
   difference of two input TIFFs.
 - Add a viewer for examining the differences between rendered
   output and golden images.
 - The viewer consists of a simple server of web API endpoints
   for querying difference results (along with rendered images
   in TIFF).
 - And a web-based (html + lit-element) UI for looking at the
   rendered images and differences.

test/renderdiff/src/compare.py

@@ -4,26 +4,54 @@ import sys
 import pprint
 import json
 
-from utils import execute, ArgParseImpl, important_print
-from image_diff import same_image
+from utils import execute, ArgParseImpl, important_print, mkdir_p
+from image_diff import same_image, output_image_diff
 from results import RESULT_OK, RESULT_FAILED, RESULT_MISSING
 
-def _compare_goldens(base_dir, comparison_dir):
-  render_results = {}
-  base_files = glob.glob(os.path.join(base_dir, "./**/*.tif"))
-  for golden_file in base_files:
-    base_fname = os.path.abspath(golden_file)
-    test_case = base_fname.replace(f'{os.path.abspath(base_dir)}/', '')
-    comp_fname = os.path.abspath(os.path.join(comparison_dir, test_case))
-    if not os.path.exists(comp_fname):
-      print(f'file name not found: {comp_fname}')
-      render_results[test_case] = RESULT_MISSING
-      continue
-    if not same_image(base_fname, comp_fname):
-      render_results[test_case] = RESULT_FAILED
-    else:
-      render_results[test_case] = RESULT_OK
-  return render_results
+def _compare_goldens(base_dir, comparison_dir, out_dir=None):
+  all_files = glob.glob(os.path.join(base_dir, "./**/*.tif"), recursive=True)
+  test_dirs = set(os.path.abspath(os.path.dirname(f)).replace(os.path.abspath(base_dir) + '/', '') \
+                  for f in all_files)
+  all_results = []
+  for test_dir in test_dirs:
+    results_meta = {}
+    results = []
+    output_test_dir = None if not out_dir else os.path.join(out_dir, test_dir)
+    if output_test_dir:
+      mkdir_p(output_test_dir)
+    base_test_dir = os.path.abspath(os.path.join(base_dir, test_dir))
+    comp_test_dir = os.path.abspath(os.path.join(comparison_dir, test_dir))
+    results_meta['base_dir'] = base_test_dir
+    results_meta['comparison_dir'] = comp_test_dir
+    for golden_file in \
+        glob.glob(os.path.join(base_test_dir, "*.tif")):
+      base_fname = os.path.abspath(golden_file)
+      test_case = base_fname.replace(f'{base_test_dir}/', '')
+      comp_fname = os.path.join(comp_test_dir, test_case)
+      result = {
+        'name': test_case,
+      }
+      if not os.path.exists(comp_fname):
+        print(f'file name not found: {comp_fname}')
+        result['result'] = RESULT_MISSING
+      elif not same_image(base_fname, comp_fname):
+        result['result'] = RESULT_FAILED
+        if output_test_dir:
+          # just the file name
+          diff_fname = f"{test_case.replace('.tif', '_diff.tif')}"
+          output_image_diff(base_fname, comp_fname, os.path.join(output_test_dir, diff_fname))
+          result['diff'] = diff_fname
+      else:
+        result['result'] = RESULT_OK
+      results.append(result)
+    if output_test_dir:
+      results_meta['results'] = results
+      output_fname = os.path.join(output_test_dir, "compare_results.json")
+      with open(output_fname, 'w') as f:
+        f.write(json.dumps(results_meta, indent=2))
+      important_print(f'Written comparison results for {test_dir} to \n    {output_fname}')
+    all_results += results
+  return all_results
 
 if __name__ == '__main__':
   parser = ArgParseImpl()
@@ -39,14 +67,9 @@ if __name__ == '__main__':
     dest = os.path.join(os.getcwd(), './out/renderdiff_tests')
   assert os.path.exists(dest), f"Destination folder={dest} does not exist."
 
-  results = _compare_goldens(args.src, dest)
+  results = _compare_goldens(args.src, dest, out_dir=args.out)
 
-  if args.out:
-    assert os.path.exists(arg.out), f"Output folder={dest} does not exist."
-    with open(os.path.join(args.out, "compare_results.json", 'w')) as f:
-      f.write(json.dumps(results))
-
-  failed = [f"   {k}" for k in results.keys() if results[k] != RESULT_OK]
+  failed = [f"   {k['name']}" for k in results if k['result'] != RESULT_OK]
   success_count = len(results) - len(failed)
   important_print(f'Successfully compared {success_count} / {len(results)} images' +
                     ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))