Implement user-specified pixel shaders, redux (#8565)

Co-authored-by: mrange <marten_range@hotmail.com>

I loved the pixel shaders in #7058, but that PR needed a bit of polish
to be ready for ingestion. This PR is almost _exactly_ that PR, with
some small changes.

* It adds a new pre-profile setting `"experimental.pixelShaderPath"`,
  which lets the user set a pixel shader to use with the Terminal.
    - CHANGED FROM #7058: It does _not_ add any built-in shaders.
    - CHANGED FROM #7058: it will _override_
      `experimental.retroTerminalEffect`
* It adds a bunch of sample shaders in `samples/shaders`. Included: 
    - A NOP shader as a base to build from.
    - An "invert" shader that inverts the colors, as a simple example
    - An "grayscale" shader that converts all colors to grayscale, as a
      simple example
    - An "raster bars" shader that draws some colored bars on the screen
      with a drop shadow, as a more involved example
    - The original retro terminal effects, as a more involved example
    - It also includes a broken shader, as an example of what heppens
      when the shader fails to compile
    - CHANGED FROM #7058: It does _not_ add the "retroII" shader we were
      all worried about.
* When a shader fails to be found or fails to compile, we'll display an
  error dialog to the user with a relevant error message.
    - CHANGED FROM #7058: Originally, #7058 would display "error bars"
      on the screen. I've removed that, and had the Terminal disable the
      shader entirely then.
* Renames the `toggleRetroEffect` action to `toggleShaderEffect`.
  (`toggleRetroEffect` is now an alias to `toggleShaderEffect`). This
  action will turn the shader OR the retro effects on/off. 

`toggleShaderEffect` works the way you'd expect it to, but the mental
math on _how_ is a little weird. The logic is basically:

```
useShader = shaderEffectsEnabled ? 
                (pixelShaderProvided ? 
                    pixelShader : 
                    (retroEffectEnabled ? 
                        retroEffect : null
                    )
                ) : 
                null
```

and `toggleShaderEffect` toggles `shaderEffectsEnabled`.

* If you've got both a shader and retro enabled, `toggleShaderEffect`
  will toggle between the shader on/off.
* If you've got a shader and retro disabled, `toggleShaderEffect` will
  toggle between the shader on/off.

References #6191
References #7058

Closes #7013

Closes #3930 "Add setting to retro terminal shader to control blur
radius, color" 
Closes #3929 "Add setting to retro terminal shader to enable drawing
scanlines" 
     - At this point, just roll your own version of the shader.

.github/actions/spell-check/dictionary/apis.txt

@@ -8,9 +8,12 @@ cmdletbinding
 COLORPROPERTY
 CXICON
 CYICON
+D2DERR_SHADER_COMPILE_FAILED
+DERR
 environstrings
 EXPCMDFLAGS
 EXPCMDSTATE
+frac
 fullkbd
 futex
 GETDESKWALLPAPER
@@ -34,7 +37,6 @@ IObject
 IStorage
 ITab
 ITaskbar
-llabs
 LCID
 llabs
 localtime
@@ -64,6 +66,7 @@ semver
 serializer
 shobjidl
 SIZENS
+smoothstep
 GETDESKWALLPAPER
 snprintf
 spsc
@@ -83,6 +86,7 @@ UPDATEINIFILE
 userenv
 wcsstr
 wcstoui
+wpc
 wsregex
 XDocument
 XElement

.github/actions/spell-check/dictionary/names.txt

@@ -12,16 +12,20 @@ ekg
 ethanschoonover
 Firefox
 Gatta
+glsl
 Grie
 Griese
 Hernan
 Howett
 Illhardt
+iquilezles
 jantari
 jerrysh
 Kaiyu
 kimwalisch
 KMehrain
+KODELIFE
+Kodelife
 Kourosh
 kowalczyk
 leonmsft
@@ -30,6 +34,7 @@ lukesampson
 Manandhar
 mbadolato
 Mehrain
+menger
 mgravell
 michaelniksa
 michkap
@@ -43,6 +48,7 @@ nvaccess
 nvda
 oising
 oldnewthing
+opengl
 osgwiki
 paulcam
 pauldotknopf
@@ -51,6 +57,7 @@ Pham
 Rincewind
 rprichard
 Schoonover
+shadertoy
 Somuah
 sonph
 sonpham
@@ -60,8 +67,8 @@ Wirt
 Wojciech
 zadjii
 Zamor
-Zamora
 zamora
+Zamora
 Zoey
 zorio
 Zverovich

.github/actions/spell-check/expect/expect.txt

@@ -24,8 +24,8 @@ ADDREF
 addressof
 ADDSTRING
 ADDTOOL
-AEnd
 aef
+AEnd
 AFew
 AFill
 AFX
@@ -628,6 +628,7 @@ doskey
 dotnet
 doubleclick
 downlevel
+DOWNSCALE
 dpg
 dpi
 DPIAPI
@@ -832,6 +833,7 @@ fuzzwrapper
 fwdecl
 fwe
 fwlink
+GAUSSIAN
 gb
 gci
 gcx
@@ -909,6 +911,7 @@ Goldmine
 gonce
 Google
 goutput
+GPUs
 Gravell's
 grayscale
 GREENSCROLL
@@ -1109,6 +1112,7 @@ INTERCEPTCOPYPASTE
 INTERNALNAME
 interop
 interoperability
+intersectors
 inthread
 intptr
 intsafe
@@ -1207,6 +1211,7 @@ KJ
 KLF
 KLMNOPQRST
 KLMNOPQRSTQQQQQ
+Kode
 KU
 KVM
 KX
@@ -1591,6 +1596,7 @@ OACR
 oauth
 objbase
 ocf
+ocolor
 odl
 oem
 oemcp
@@ -1789,6 +1795,7 @@ pragma
 prc
 prealigned
 prebuilt
+precendence
 precomp
 prect
 prefast
@@ -1834,6 +1841,7 @@ pshn
 PSHNOTIFY
 PSHORT
 pshpack
+psin
 PSINGLE
 psl
 psldl
@@ -1882,10 +1890,12 @@ QWER
 qzmp
 RAII
 RALT
+rasterbar
 rasterfont
 rasterization
 rawinput
 RAWPATH
+raytracers
 razzlerc
 rbar
 rbegin
@@ -1956,9 +1966,11 @@ RESETCONTENT
 resheader
 resizable
 resmimetype
+reso
 restrictedcapabilities
 resw
 resx
+RETROII
 retval
 rfa
 rfc
@@ -2029,11 +2041,14 @@ SBCSDBCS
 sbi
 sbiex
 sbold
+sbri
+scanbri
 scancode
 scanline
 schemename
 SCL
 scm
+scol
 scprintf
 SCRBUF
 SCRBUFSIZE
@@ -2167,6 +2182,9 @@ SOURCESDIRECTORY
 SPACEBAR
 spammy
 spand
+spe
+sph
+spherefunctions
 splashscreen
 sprintf
 sqlproj
@@ -2296,6 +2314,7 @@ tellp
 telnet
 telnetd
 templated
+teraflop
 terminalcore
 TERMINALSCROLLING
 terminfo
@@ -2544,6 +2563,7 @@ vga
 vgaoem
 viewkind
 viewports
+Viginetting
 Virt
 VIRTTERM
 Virtualizing
@@ -2760,6 +2780,7 @@ wwaproj
 WWith
 wx
 wxh
+wz
 xa
 xact
 xamarin
@@ -2809,7 +2830,10 @@ xutr
 xvalue
 XVIRTUALSCREEN
 XWalk
+XWV
 xy
+xyw
+Xzn
 yact
 YAML
 YCast
@@ -2825,6 +2849,7 @@ YVIRTUALSCREEN
 Yw
 YWalk
 yx
+yzx
 Zc
 ZCmd
 ZCtrl

doc/cascadia/profiles.schema.json

@@ -103,7 +103,7 @@
         "toggleFocusMode",
         "toggleFullscreen",
         "togglePaneZoom",
-        "toggleRetroEffect",
+        "toggleShaderEffects",
         "wt",
         "unbound"
       ],
@@ -936,6 +936,10 @@
           "description": "When set to true, enable retro terminal effects. This is an experimental feature, and its continued existence is not guaranteed.",
           "type": "boolean"
         },
+        "experimental.pixelShaderPath": {
+          "description": "Use to set a path to a pixel shader to use with the Terminal. Overrides `experimental.retroTerminalEffect`. This is an experimental feature, and its continued existence is not guaranteed.",
+          "type": "string"
+        },
         "fontFace": {
           "default": "Cascadia Mono",
           "description": "Name of the font face used in the profile.",

samples/PixelShaders/Broken.hlsl

@@ -0,0 +1,18 @@
+// Broken, can be used for explorative testing of pixel shader error handling
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+cbuffer PixelShaderSettings {
+  float  Time;
+  float  Scale;
+  float2 Resolution;
+  float4 Background;
+};
+
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    // OOPS; vec4 is not a hlsl but a glsl datatype!
+    vec4 color = shaderTexture.Sample(samplerState, tex);
+
+    return color;
+}

samples/PixelShaders/Error.hlsl

@@ -0,0 +1,18 @@
+// Shader used to indicate something went wrong during shader loading
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+cbuffer PixelShaderSettings {
+  float  Time;
+  float  Scale;
+  float2 Resolution;
+  float4 Background;
+};
+
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    float4 color = shaderTexture.Sample(samplerState, tex);
+    float bars = 0.5+0.5*sin(tex.y*100);
+    color.x += pow(bars, 20.0);
+    return color;
+}

samples/PixelShaders/Grayscale.hlsl

@@ -0,0 +1,32 @@
+// A minimal pixel shader that inverts the colors
+
+// The terminal graphics as a texture
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+// Terminal settings such as the resolution of the texture
+cbuffer PixelShaderSettings {
+  // Time since pixel shader was enabled
+  float  Time;
+  // UI Scale
+  float  Scale;
+  // Resolution of the shaderTexture
+  float2 Resolution;
+  // Background color as rgba
+  float4 Background;
+};
+
+// A pixel shader is a program that given a texture coordinate (tex) produces a color
+//  Just ignore the pos parameter
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    // Read the color value at the current texture coordinate (tex)
+    //  float4 is tuple of 4 floats, rgba
+    float4 color = shaderTexture.Sample(samplerState, tex);
+    float avg = (color.x + color.y + color.z) / 3.0;
+    // Inverts the rgb values (xyz) but don't touch the alpha (w)
+    color.xyz = avg;
+
+    // Return the final color
+    return color;
+}

samples/PixelShaders/Invert.hlsl

@@ -0,0 +1,32 @@
+// A minimal pixel shader that inverts the colors
+
+// The terminal graphics as a texture
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+// Terminal settings such as the resolution of the texture
+cbuffer PixelShaderSettings {
+  // Time since pixel shader was enabled
+  float  Time;
+  // UI Scale
+  float  Scale;
+  // Resolution of the shaderTexture
+  float2 Resolution;
+  // Background color as rgba
+  float4 Background;
+};
+
+// A pixel shader is a program that given a texture coordinate (tex) produces a color
+//  Just ignore the pos parameter
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    // Read the color value at the current texture coordinate (tex)
+    //  float4 is tuple of 4 floats, rgba
+    float4 color = shaderTexture.Sample(samplerState, tex);
+
+    // Inverts the rgb values (xyz) but don't touch the alpha (w)
+    color.xyz = 1.0 - color.xyz;
+
+    // Return the final color
+    return color;
+}

samples/PixelShaders/Nop.hlsl

@@ -0,0 +1,17 @@
+// Does nothing, serves as an example of a minimal pixel shader
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+cbuffer PixelShaderSettings {
+  float  Time;
+  float  Scale;
+  float2 Resolution;
+  float4 Background;
+};
+
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    float4 color = shaderTexture.Sample(samplerState, tex);
+
+    return color;
+}

samples/PixelShaders/README.md

@@ -0,0 +1,136 @@
+# Pixel Shaders in Windows Terminal
+
+Due to the sheer amount of computing power in GPUs, one can do awesome things with pixel shaders such as real-time fractal zoom, ray tracers and image processing.
+
+Windows Terminal allows user to provide a pixel shader which will be applied to the terminal. To try it out, add the following setting to one of your profiles:
+
+```
+"experimental.pixelShaderPath": "<path to a .hlsl pixel shader>"
+```
+> **Note**: if you specify a shader with `experimental.pixelShaderPath`, the Terminal will use that instead of the `experimental.retroTerminalEffect`.
+
+To get started using pixel shaders in the Terminal, start with the following sample shader. This is `Invert.hlsl` in this directory:
+
+```hlsl
+// A minimal pixel shader that inverts the colors
+
+// The terminal graphics as a texture
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+// Terminal settings such as the resolution of the texture
+cbuffer PixelShaderSettings {
+  // Time since pixel shader was enabled
+  float  Time;
+  // UI Scale
+  float  Scale;
+  // Resolution of the shaderTexture
+  float2 Resolution;
+  // Background color as rgba
+  float4 Background;
+};
+
+// A pixel shader is a program that given a texture coordinate (tex) produces a color
+//  Just ignore the pos parameter
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    // Read the color value at the current texture coordinate (tex)
+    //  float4 is tuple of 4 floats, rgba
+    float4 color = shaderTexture.Sample(samplerState, tex);
+
+    // Inverts the rgb values (xyz) but don't touch the alpha (w)
+    color.xyz = 1.0 - color.xyz;
+
+    // Return the final color
+    return color;
+}
+```
+
+Save this file as `C:\temp\invert.hlsl`, then update a profile with the setting:
+
+```
+"experimental.pixelShaderEffect": "C:\\temp\\invert.hlsl"
+```
+
+Once the settings file is saved, open a terminal with the changed profile. It should now invert the colors of the screen!
+
+If your shader fails to compile, the Terminal will display a warning dialog and ignore it temporarily. After fixing your shader, touch the `settings.json` file again, or open a new tab, and the Terminal will try loading the shader again.
+
+## HLSL
+
+The language we use to write pixel shaders is called `HLSL`. It a `C`-like language, with some restrictions.You can't allocate memory, use pointers or recursion.
+What you get access to is computing power in the teraflop range on decently recent GPUs. This means writing real-time raytracers or other cool effects are in the realm of possibility.
+
+[shadertoy](https://shadertoy.com/) is a great site that show case what's possible with pixel shaders (albeit in `GLSL`). For example this [menger sponge](https://www.shadertoy.com/view/4scXzn). Converting from `GLSL` to `HLSL` isn't overly hard once you gotten the hang of it.
+
+## Adding some retro raster bars
+
+Let's try a more complicated example. Raster bars was cool in the 80's, so let's add that. Start by modifying shader like so: (This is `Rasterbars.hlsl`)
+
+```hlsl
+// A minimal pixel shader that shows some raster bars
+
+// The terminal graphics as a texture
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+// Terminal settings such as the resolution of the texture
+cbuffer PixelShaderSettings {
+  // Time since pixel shader was enabled
+  float  Time;
+  // UI Scale
+  float  Scale;
+  // Resolution of the shaderTexture
+  float2 Resolution;
+  // Background color as rgba
+  float4 Background;
+};
+
+// A pixel shader is a program that given a texture coordinate (tex) produces a color
+//  Just ignore the pos parameter
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    // Read the color value at the current texture coordinate (tex)
+    //  float4 is tuple of 4 floats, rgba
+    float4 color = shaderTexture.Sample(samplerState, tex);
+
+    // Read the color value at some offset, will be used as shadow
+    float4 ocolor = shaderTexture.Sample(samplerState, tex+2.0*Scale*float2(-1.0, -1.0)/Resolution.y);
+
+    // Thickness of raster
+    const float thickness = 0.1;
+
+    float ny = floor(tex.y/thickness);
+    float my = tex.y%thickness;
+    const float pi = 3.141592654;
+
+
+    // ny is used to compute the rasterbar base color
+    float cola = ny*2.0*pi;
+    float3 col = 0.75+0.25*float3(sin(cola*0.111), sin(cola*0.222), sin(cola*0.333));
+
+    // my is used to compute the rasterbar brightness
+    //  smoothstep is a great little function: https://en.wikipedia.org/wiki/Smoothstep
+    float brightness = 1.0-smoothstep(0.0, thickness*0.5, abs(my - 0.5*thickness));
+
+    float3 rasterColor = col*brightness;
+
+    // lerp(x, y, a) is another very useful function: https://en.wikipedia.org/wiki/Linear_interpolation
+    float3 final = rasterColor;
+    // Create the drop shadow of the terminal graphics
+    //  .w is the alpha channel, 0 is fully transparent and 1 is fully opaque
+    final = lerp(final, float(0.0), ocolor.w);
+    // Draw the terminal graphics
+    final = lerp(final, color.xyz, color.w);
+
+    // Return the final color, set alpha to 1 (ie opaque)
+    return float4(final, 1.0);
+}
+```
+
+Once reloaded, it should show some retro raster bars in the background, with a drop shadow to make the text more readable.
+
+## Retro Terminal Effect
+
+As a more complicated example, the Terminal's built-in `experimental.retroTerminalEffect` is included as the `Retro.hlsl` file in this directory. Feel free to modify and experiment!
+

samples/PixelShaders/Rasterbars.hlsl

@@ -0,0 +1,58 @@
+// A minimal pixel shader that shows some raster bars
+
+// The terminal graphics as a texture
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+// Terminal settings such as the resolution of the texture
+cbuffer PixelShaderSettings {
+  // Time since pixel shader was enabled
+  float  Time;
+  // UI Scale
+  float  Scale;
+  // Resolution of the shaderTexture
+  float2 Resolution;
+  // Background color as rgba
+  float4 Background;
+};
+
+// A pixel shader is a program that given a texture coordinate (tex) produces a color
+//  Just ignore the pos parameter
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    // Read the color value at the current texture coordinate (tex)
+    //  float4 is tuple of 4 floats, rgba
+    float4 color = shaderTexture.Sample(samplerState, tex);
+
+    // Read the color value at some offset, will be used as shadow
+    float4 ocolor = shaderTexture.Sample(samplerState, tex+2.0*Scale*float2(-1.0, -1.0)/Resolution.y);
+
+    // Thickness of raster
+    const float thickness = 0.1;
+
+    float ny = floor(tex.y/thickness);
+    float my = tex.y%thickness;
+    const float pi = 3.141592654;
+
+
+    // ny is used to compute the rasterbar base color
+    float cola = ny*2.0*pi;
+    float3 col = 0.75+0.25*float3(sin(cola*0.111), sin(cola*0.222), sin(cola*0.333));
+
+    // my is used to compute the rasterbar brightness
+    //  smoothstep is a great little function: https://en.wikipedia.org/wiki/Smoothstep
+    float brightness = 1.0-smoothstep(0.0, thickness*0.5, abs(my - 0.5*thickness));
+
+    float3 rasterColor = col*brightness;
+
+    // lerp(x, y, a) is another very useful function: https://en.wikipedia.org/wiki/Linear_interpolation
+    float3 final = rasterColor;
+    // Create the drop shadow of the terminal graphics
+    //  .w is the alpha channel, 0 is fully transparent and 1 is fully opaque
+    final = lerp(final, float(0.0), ocolor.w);
+    // Draw the terminal graphics
+    final = lerp(final, color.xyz, color.w);
+
+    // Return the final color, set alpha to 1 (ie opaque)
+    return float4(final, 1.0);
+}

samples/PixelShaders/Retro.hlsl

@@ -0,0 +1,83 @@
+// The original retro pixel shader
+Texture2D shaderTexture;
+SamplerState samplerState;
+
+cbuffer PixelShaderSettings {
+  float  Time;
+  float  Scale;
+  float2 Resolution;
+  float4 Background;
+};
+
+#define SCANLINE_FACTOR 0.5
+#define SCALED_SCANLINE_PERIOD Scale
+#define SCALED_GAUSSIAN_SIGMA (2.0*Scale)
+
+static const float M_PI = 3.14159265f;
+
+float Gaussian2D(float x, float y, float sigma)
+{
+    return 1/(sigma*sqrt(2*M_PI)) * exp(-0.5*(x*x + y*y)/sigma/sigma);
+}
+
+float4 Blur(Texture2D input, float2 tex_coord, float sigma)
+{
+    uint width, height;
+    shaderTexture.GetDimensions(width, height);
+
+    float texelWidth = 1.0f/width;
+    float texelHeight = 1.0f/height;
+
+    float4 color = { 0, 0, 0, 0 };
+
+    int sampleCount = 13;
+
+    for (int x = 0; x < sampleCount; x++)
+    {
+        float2 samplePos = { 0, 0 };
+
+        samplePos.x = tex_coord.x + (x - sampleCount/2) * texelWidth;
+        for (int y = 0; y < sampleCount; y++)
+        {
+            samplePos.y = tex_coord.y + (y - sampleCount/2) * texelHeight;
+            if (samplePos.x <= 0 || samplePos.y <= 0 || samplePos.x >= width || samplePos.y >= height) continue;
+
+            color += input.Sample(samplerState, samplePos) * Gaussian2D((x - sampleCount/2), (y - sampleCount/2), sigma);
+        }
+    }
+
+    return color;
+}
+
+float SquareWave(float y)
+{
+    return 1 - (floor(y / SCALED_SCANLINE_PERIOD) % 2) * SCANLINE_FACTOR;
+}
+
+float4 Scanline(float4 color, float4 pos)
+{
+    float wave = SquareWave(pos.y);
+
+    // TODO:GH#3929 make this configurable.
+    // Remove the && false to draw scanlines everywhere.
+    if (length(color.rgb) < 0.2 && false)
+    {
+        return color + wave*0.1;
+    }
+    else
+    {
+        return color * wave;
+    }
+}
+
+float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
+{
+    Texture2D input = shaderTexture;
+
+    // TODO:GH#3930 Make these configurable in some way.
+    float4 color = input.Sample(samplerState, tex);
+    color += Blur(input, tex, SCALED_GAUSSIAN_SIGMA)*0.3;
+    color = Scanline(color, pos);
+
+    return color;
+}

src/cascadia/TerminalApp/AppActionHandlers.cpp

@@ -295,11 +295,11 @@ namespace winrt::TerminalApp::implementation
         args.Handled(true);
     }
 
-    void TerminalPage::_HandleToggleRetroEffect(const IInspectable& /*sender*/,
+    void TerminalPage::_HandleToggleShaderEffects(const IInspectable& /*sender*/,
-                                                const ActionEventArgs& args)
+                                                  const ActionEventArgs& args)
     {
         const auto termControl = _GetActiveControl();
-        termControl.ToggleRetroEffect();
+        termControl.ToggleShaderEffects();
         args.Handled(true);
     }
 

src/cascadia/TerminalApp/ShortcutActionDispatch.cpp

@@ -176,9 +176,9 @@ namespace winrt::TerminalApp::implementation
             _ResetFontSizeHandlers(*this, eventArgs);
             break;
         }
-        case ShortcutAction::ToggleRetroEffect:
+        case ShortcutAction::ToggleShaderEffects:
         {
-            _ToggleRetroEffectHandlers(*this, eventArgs);
+            _ToggleShaderEffectsHandlers(*this, eventArgs);
             break;
         }
         case ShortcutAction::ToggleFocusMode:

src/cascadia/TerminalApp/ShortcutActionDispatch.h

@@ -49,7 +49,7 @@ namespace winrt::TerminalApp::implementation
         TYPED_EVENT(ResizePane,           TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);
         TYPED_EVENT(Find,                 TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);
         TYPED_EVENT(MoveFocus,            TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);
-        TYPED_EVENT(ToggleRetroEffect,    TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);
+        TYPED_EVENT(ToggleShaderEffects,  TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);
         TYPED_EVENT(ToggleFocusMode,      TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);
         TYPED_EVENT(ToggleFullscreen,     TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);
         TYPED_EVENT(ToggleAlwaysOnTop,    TerminalApp::ShortcutActionDispatch, Microsoft::Terminal::Settings::Model::ActionEventArgs);

src/cascadia/TerminalApp/ShortcutActionDispatch.idl

@@ -35,7 +35,7 @@ namespace TerminalApp
         event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> ResizePane;
         event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> Find;
         event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> MoveFocus;
-        event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> ToggleRetroEffect;
+        event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> ToggleShaderEffects;
         event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> ToggleFocusMode;
         event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> ToggleFullscreen;
         event Windows.Foundation.TypedEventHandler<ShortcutActionDispatch, Microsoft.Terminal.Settings.Model.ActionEventArgs> ToggleAlwaysOnTop;

src/cascadia/TerminalApp/TerminalPage.cpp

@@ -976,7 +976,7 @@ namespace winrt::TerminalApp::implementation
         _actionDispatch->AdjustFontSize({ this, &TerminalPage::_HandleAdjustFontSize });
         _actionDispatch->Find({ this, &TerminalPage::_HandleFind });
         _actionDispatch->ResetFontSize({ this, &TerminalPage::_HandleResetFontSize });
-        _actionDispatch->ToggleRetroEffect({ this, &TerminalPage::_HandleToggleRetroEffect });
+        _actionDispatch->ToggleShaderEffects({ this, &TerminalPage::_HandleToggleShaderEffects });
         _actionDispatch->ToggleFocusMode({ this, &TerminalPage::_HandleToggleFocusMode });
         _actionDispatch->ToggleFullscreen({ this, &TerminalPage::_HandleToggleFullscreen });
         _actionDispatch->ToggleAlwaysOnTop({ this, &TerminalPage::_HandleToggleAlwaysOnTop });

src/cascadia/TerminalApp/TerminalPage.h

@@ -282,7 +282,7 @@ namespace winrt::TerminalApp::implementation
         void _HandleAdjustFontSize(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);
         void _HandleFind(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);
         void _HandleResetFontSize(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);
-        void _HandleToggleRetroEffect(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);
+        void _HandleToggleShaderEffects(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);
         void _HandleToggleFocusMode(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);
         void _HandleToggleFullscreen(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);
         void _HandleToggleAlwaysOnTop(const IInspectable& sender, const Microsoft::Terminal::Settings::Model::ActionEventArgs& args);

src/cascadia/TerminalApp/TerminalSettings.cpp

@@ -182,6 +182,7 @@ namespace winrt::TerminalApp::implementation
         std::tie(_BackgroundImageHorizontalAlignment, _BackgroundImageVerticalAlignment) = ConvertConvergedAlignment(profile.BackgroundImageAlignment());
 
         _RetroTerminalEffect = profile.RetroTerminalEffect();
+        _PixelShaderPath = winrt::hstring{ wil::ExpandEnvironmentStringsW<std::wstring>(profile.PixelShaderPath().c_str()) };
 
         _AntialiasingMode = profile.AntialiasingMode();
 

src/cascadia/TerminalApp/TerminalSettings.h

@@ -121,6 +121,7 @@ namespace winrt::TerminalApp::implementation
         GETSET_PROPERTY(bool, SoftwareRendering, false);
         GETSET_PROPERTY(bool, ForceVTInput, false);
 
+        GETSET_PROPERTY(hstring, PixelShaderPath);
 #pragma warning(pop)
 
     private:

src/cascadia/TerminalControl/IControlSettings.idl

@@ -54,8 +54,11 @@ namespace Microsoft.Terminal.TerminalControl
 
         TextAntialiasingMode AntialiasingMode;
 
+        // Experimental Settings
         Boolean RetroTerminalEffect;
         Boolean ForceFullRepaintRendering;
         Boolean SoftwareRendering;
+        String PixelShaderPath;
+
     };
 }

src/cascadia/TerminalControl/Resources/en-US/Resources.resw

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!-- 
+  <!--
-    Microsoft ResX Schema 
+    Microsoft ResX Schema
-    
+
     Version 2.0
-    
+
-    The primary goals of this format is to allow a simple XML format 
+    The primary goals of this format is to allow a simple XML format
-    that is mostly human readable. The generation and parsing of the 
+    that is mostly human readable. The generation and parsing of the
-    various data types are done through the TypeConverter classes 
+    various data types are done through the TypeConverter classes
     associated with the data types.
-    
+
     Example:
-    
+
     ... ado.net/XML headers & schema ...
     <resheader name="resmimetype">text/microsoft-resx</resheader>
     <resheader name="version">2.0</resheader>
@@ -26,36 +26,36 @@
         <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
         <comment>This is a comment</comment>
     </data>
-                
+
-    There are any number of "resheader" rows that contain simple 
+    There are any number of "resheader" rows that contain simple
     name/value pairs.
-    
+
-    Each data row contains a name, and value. The row also contains a 
+    Each data row contains a name, and value. The row also contains a
-    type or mimetype. Type corresponds to a .NET class that support 
+    type or mimetype. Type corresponds to a .NET class that support
-    text/value conversion through the TypeConverter architecture. 
+    text/value conversion through the TypeConverter architecture.
-    Classes that don't support this are serialized and stored with the 
+    Classes that don't support this are serialized and stored with the
     mimetype set.
-    
+
-    The mimetype is used for serialized objects, and tells the 
+    The mimetype is used for serialized objects, and tells the
-    ResXResourceReader how to depersist the object. This is currently not 
+    ResXResourceReader how to depersist the object. This is currently not
     extensible. For a given mimetype the value must be set accordingly:
-    
+
-    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    Note - application/x-microsoft.net.object.binary.base64 is the format
-    that the ResXResourceWriter will generate, however the reader can 
+    that the ResXResourceWriter will generate, however the reader can
     read any of the formats listed below.
-    
+
     mimetype: application/x-microsoft.net.object.binary.base64
-    value   : The object must be serialized with 
+    value   : The object must be serialized with
             : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
             : and then encoded with base64 encoding.
-    
+
     mimetype: application/x-microsoft.net.object.soap.base64
-    value   : The object must be serialized with 
+    value   : The object must be serialized with
             : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
             : and then encoded with base64 encoding.
 
     mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array 
+    value   : The object must be serialized into a byte array
             : using a System.ComponentModel.TypeConverter
             : and then encoded with base64 encoding.
     -->
@@ -179,11 +179,22 @@
     <value>Ctrl+Click to follow link</value>
   </data>
   <data name="NoticeFontNotFound" xml:space="preserve">
-    <value>Unable to find the selected font "{0}". 
+    <value>Unable to find the selected font "{0}".
 
-"{1}" has been selected instead. 
+"{1}" has been selected instead.
 
 Please either install the missing font or choose another one.</value>
     <comment>0 and 1 are names of fonts provided by the user and system respectively.</comment>
   </data>
-</root>
+  <data name="PixelShaderNotFound" xml:space="preserve">
+    <value>Unable to find the provided shader "{0}".</value>
+    <comment>{0} is a file name</comment>
+  </data>
+  <data name="PixelShaderCompileFailed" xml:space="preserve">
+    <value>Unable to compile the specified pixel shader.</value>
+  </data>
+  <data name="UnexpectedRendererError" xml:space="preserve">
+    <value>Renderer encountered an unexpected error: {0}</value>
+    <comment>{0} is an error code.</comment>
+  </data>
+</root>

src/cascadia/TerminalControl/TermControl.cpp

@@ -308,7 +308,8 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
             // Update DxEngine settings under the lock
             _renderEngine->SetSelectionBackground(_settings.SelectionBackground());
 
-            _renderEngine->SetRetroTerminalEffects(_settings.RetroTerminalEffect());
+            _renderEngine->SetRetroTerminalEffect(_settings.RetroTerminalEffect());
+            _renderEngine->SetPixelShaderPath(_settings.PixelShaderPath());
             _renderEngine->SetForceFullRepaintRendering(_settings.ForceFullRepaintRendering());
             _renderEngine->SetSoftwareRendering(_settings.SoftwareRendering());
 
@@ -348,10 +349,27 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         _SendInputToConnection(wstr);
     }
 
-    void TermControl::ToggleRetroEffect()
+    void TermControl::ToggleShaderEffects()
     {
         auto lock = _terminal->LockForWriting();
-        _renderEngine->SetRetroTerminalEffects(!_renderEngine->GetRetroTerminalEffects());
+        // Originally, this action could be used to enable the retro effects
+        // even when they're set to `false` in the settings. If the user didn't
+        // specify a custom pixel shader, manually enable the legacy retro
+        // effect first. This will ensure that a toggle off->on will still work,
+        // even if they currently have retro effect off.
+        if (_settings.PixelShaderPath().empty() && !_renderEngine->GetRetroTerminalEffect())
+        {
+            // SetRetroTerminalEffect to true will enable the effect. In this
+            // case, the shader effect will already be disabled (because neither
+            // a pixel shader nor the retro effects were originally requested).
+            // So we _don't_ want to toggle it again below, because that would
+            // toggle it back off.
+            _renderEngine->SetRetroTerminalEffect(true);
+        }
+        else
+        {
+            _renderEngine->ToggleShaderEffects();
+        }
     }
 
     // Method Description:
@@ -616,6 +634,45 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         }
     }
 
+    // Method Description:
+    // - Called when the renderer triggers a warning. It might do this when it
+    //   fails to find a shader file, or fails to compile a shader. We'll take
+    //   that renderer warning, and display a dialog to the user with and
+    //   appropriate error message. WE'll display the dialog with our
+    //   RaiseNotice event.
+    // Arguments:
+    // - hr: an  HRESULT describing the warning
+    // Return Value:
+    // - <none>
+    winrt::fire_and_forget TermControl::_RendererWarning(const HRESULT hr)
+    {
+        auto weakThis{ get_weak() };
+        co_await winrt::resume_foreground(Dispatcher());
+
+        if (auto control{ weakThis.get() })
+        {
+            winrt::hstring message;
+            if (HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND) == hr ||
+                HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND) == hr)
+            {
+                message = { fmt::format(std::wstring_view{ RS_(L"PixelShaderNotFound") },
+                                        _settings.PixelShaderPath()) };
+            }
+            else if (D2DERR_SHADER_COMPILE_FAILED == hr)
+            {
+                message = { fmt::format(std::wstring_view{ RS_(L"PixelShaderCompileFailed") }) };
+            }
+            else
+            {
+                message = { fmt::format(std::wstring_view{ RS_(L"UnexpectedRendererError") },
+                                        hr) };
+            }
+
+            auto noticeArgs = winrt::make<NoticeEventArgs>(NoticeLevel::Warning, std::move(message));
+            control->_raiseNoticeHandlers(*control, std::move(noticeArgs));
+        }
+    }
+
     void TermControl::_AttachDxgiSwapChainToXaml(IDXGISwapChain1* swapChain)
     {
         auto nativePanel = SwapChainPanel().as<ISwapChainPanelNative>();
@@ -695,7 +752,14 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
 
             _terminal->CreateFromSettings(_settings, renderTarget);
 
-            dxEngine->SetRetroTerminalEffects(_settings.RetroTerminalEffect());
+            // IMPORTANT! Set this callback up sooner than later. If we do it
+            // after Enable, then it'll be possible to paint the frame once
+            // _before_ the warning handler is set up, and then warnings from
+            // the first paint will be ignored!
+            dxEngine->SetWarningCallback(std::bind(&TermControl::_RendererWarning, this, std::placeholders::_1));
+
+            dxEngine->SetRetroTerminalEffect(_settings.RetroTerminalEffect());
+            dxEngine->SetPixelShaderPath(_settings.PixelShaderPath());
             dxEngine->SetForceFullRepaintRendering(_settings.ForceFullRepaintRendering());
             dxEngine->SetSoftwareRendering(_settings.SoftwareRendering());
 

src/cascadia/TerminalControl/TermControl.h

@@ -122,12 +122,13 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         void ResetFontSize();
 
         void SendInput(const winrt::hstring& input);
-        void ToggleRetroEffect();
+        void ToggleShaderEffects();
 
         winrt::fire_and_forget RenderEngineSwapChainChanged();
         void _AttachDxgiSwapChainToXaml(IDXGISwapChain1* swapChain);
         winrt::fire_and_forget _RendererEnteredErrorState();
         void _RenderRetryButton_Click(IInspectable const& button, IInspectable const& args);
+        winrt::fire_and_forget _RendererWarning(const HRESULT hr);
 
         void CreateSearchBoxControl();
 

src/cascadia/TerminalControl/TermControl.idl

@@ -17,7 +17,7 @@ namespace Microsoft.Terminal.TerminalControl
     // If you press F7 or Alt and get a runtime error, go make sure both copies are the same.
     [uuid("0ddf4edc-3fda-4dee-97ca-a417ee3dd510")] interface IDirectKeyListener {
         Boolean OnDirectKeyEvent(UInt32 vkey, UInt8 scanCode, Boolean down);
-    }
+    };
 
     [flags]
     enum CopyFormat
@@ -103,8 +103,8 @@ namespace Microsoft.Terminal.TerminalControl
         void AdjustFontSize(Int32 fontSizeDelta);
         void ResetFontSize();
 
+        void ToggleShaderEffects();
         void SendInput(String input);
-        void ToggleRetroEffect();
 
         void TaskbarProgressChanged();
         UInt64 TaskbarState { get; };

src/cascadia/TerminalSettingsModel/ActionAndArgs.cpp

@@ -47,7 +47,8 @@ static constexpr std::string_view ToggleCommandPaletteKey{ "commandPalette" };
 static constexpr std::string_view ToggleFocusModeKey{ "toggleFocusMode" };
 static constexpr std::string_view ToggleFullscreenKey{ "toggleFullscreen" };
 static constexpr std::string_view TogglePaneZoomKey{ "togglePaneZoom" };
-static constexpr std::string_view ToggleRetroEffectKey{ "toggleRetroEffect" };
+static constexpr std::string_view LegacyToggleRetroEffectKey{ "toggleRetroEffect" };
+static constexpr std::string_view ToggleShaderEffectsKey{ "toggleShaderEffects" };
 static constexpr std::string_view MoveTabKey{ "moveTab" };
 static constexpr std::string_view BreakIntoDebuggerKey{ "breakIntoDebugger" };
 
@@ -109,7 +110,8 @@ namespace winrt::Microsoft::Terminal::Settings::Model::implementation
         { ToggleFocusModeKey, ShortcutAction::ToggleFocusMode },
         { ToggleFullscreenKey, ShortcutAction::ToggleFullscreen },
         { TogglePaneZoomKey, ShortcutAction::TogglePaneZoom },
-        { ToggleRetroEffectKey, ShortcutAction::ToggleRetroEffect },
+        { LegacyToggleRetroEffectKey, ShortcutAction::ToggleShaderEffects },
+        { ToggleShaderEffectsKey, ShortcutAction::ToggleShaderEffects },
         { MoveTabKey, ShortcutAction::MoveTab },
         { BreakIntoDebuggerKey, ShortcutAction::BreakIntoDebugger },
         { UnboundKey, ShortcutAction::Invalid },
@@ -309,7 +311,7 @@ namespace winrt::Microsoft::Terminal::Settings::Model::implementation
                 { ShortcutAction::ToggleFocusMode, RS_(L"ToggleFocusModeCommandKey") },
                 { ShortcutAction::ToggleFullscreen, RS_(L"ToggleFullscreenCommandKey") },
                 { ShortcutAction::TogglePaneZoom, RS_(L"TogglePaneZoomCommandKey") },
-                { ShortcutAction::ToggleRetroEffect, RS_(L"ToggleRetroEffectCommandKey") },
+                { ShortcutAction::ToggleShaderEffects, RS_(L"ToggleShaderEffectsCommandKey") },
                 { ShortcutAction::MoveTab, L"" }, // Intentionally omitted, must be generated by GenerateName
                 { ShortcutAction::BreakIntoDebugger, RS_(L"BreakIntoDebuggerCommandKey") },
             };

src/cascadia/TerminalSettingsModel/KeyMapping.idl

@@ -36,7 +36,7 @@ namespace Microsoft.Terminal.Settings.Model
         ResizePane,
         MoveFocus,
         Find,
-        ToggleRetroEffect,
+        ToggleShaderEffects,
         ToggleFocusMode,
         ToggleFullscreen,
         ToggleAlwaysOnTop,

src/cascadia/TerminalSettingsModel/Profile.cpp

@@ -58,6 +58,7 @@ static constexpr std::string_view RetroTerminalEffectKey{ "experimental.retroTer
 static constexpr std::string_view AntialiasingModeKey{ "antialiasingMode" };
 static constexpr std::string_view TabColorKey{ "tabColor" };
 static constexpr std::string_view BellStyleKey{ "bellStyle" };
+static constexpr std::string_view PixelShaderPathKey{ "experimental.pixelShaderPath" };
 
 static constexpr std::wstring_view DesktopWallpaperEnum{ L"desktopWallpaper" };
 
@@ -109,6 +110,7 @@ winrt::com_ptr<Profile> Profile::CopySettings(winrt::com_ptr<Profile> source)
     profile->_CursorShape = source->_CursorShape;
     profile->_CursorHeight = source->_CursorHeight;
     profile->_BellStyle = source->_BellStyle;
+    profile->_PixelShaderPath = source->_PixelShaderPath;
     profile->_BackgroundImageAlignment = source->_BackgroundImageAlignment;
     profile->_ConnectionType = source->_ConnectionType;
 
@@ -334,10 +336,9 @@ void Profile::LayerJson(const Json::Value& json)
     JsonUtils::GetValueForKey(json, BackgroundImageAlignmentKey, _BackgroundImageAlignment);
     JsonUtils::GetValueForKey(json, RetroTerminalEffectKey, _RetroTerminalEffect);
     JsonUtils::GetValueForKey(json, AntialiasingModeKey, _AntialiasingMode);
-
     JsonUtils::GetValueForKey(json, TabColorKey, _TabColor);
-
     JsonUtils::GetValueForKey(json, BellStyleKey, _BellStyle);
+    JsonUtils::GetValueForKey(json, PixelShaderPathKey, _PixelShaderPath);
 }
 
 // Method Description:
@@ -529,6 +530,7 @@ Json::Value Profile::ToJson() const
     JsonUtils::SetValueForKey(json, AntialiasingModeKey, _AntialiasingMode);
     JsonUtils::SetValueForKey(json, TabColorKey, _TabColor);
     JsonUtils::SetValueForKey(json, BellStyleKey, _BellStyle);
+    JsonUtils::SetValueForKey(json, PixelShaderPathKey, _PixelShaderPath);
 
     return json;
 }

src/cascadia/TerminalSettingsModel/Profile.h

@@ -94,6 +94,7 @@ namespace winrt::Microsoft::Terminal::Settings::Model::implementation
 
         GETSET_SETTING(Microsoft::Terminal::TerminalControl::TextAntialiasingMode, AntialiasingMode, Microsoft::Terminal::TerminalControl::TextAntialiasingMode::Grayscale);
         GETSET_SETTING(bool, RetroTerminalEffect, false);
+        GETSET_SETTING(hstring, PixelShaderPath, L"");
         GETSET_SETTING(bool, ForceFullRepaintRendering, false);
         GETSET_SETTING(bool, SoftwareRendering, false);
 

src/cascadia/TerminalSettingsModel/Profile.idl

@@ -135,6 +135,10 @@ namespace Microsoft.Terminal.Settings.Model
         void ClearRetroTerminalEffect();
         Boolean RetroTerminalEffect;
 
+        Boolean HasPixelShaderPath();
+        void ClearPixelShaderPath();
+        String PixelShaderPath;
+
         Boolean HasForceFullRepaintRendering();
         void ClearForceFullRepaintRendering();
         Boolean ForceFullRepaintRendering;

src/cascadia/TerminalSettingsModel/Resources/en-US/Resources.resw

@@ -357,8 +357,8 @@
   <data name="TogglePaneZoomCommandKey" xml:space="preserve">
     <value>Toggle pane zoom</value>
   </data>
-  <data name="ToggleRetroEffectCommandKey" xml:space="preserve">
+  <data name="ToggleShaderEffectsCommandKey" xml:space="preserve">
-    <value>Toggle retro terminal effect</value>
+    <value>Toggle terminal visual effects</value>
   </data>
   <data name="BreakIntoDebuggerCommandKey" xml:space="preserve">
     <value>Break into the debugger</value>

src/cascadia/TerminalSettingsModel/defaults-universal.json

@@ -112,7 +112,6 @@
         { "command": "openSettings", "keys": "ctrl+," },
         { "command": { "action": "openSettings", "target": "defaultsFile" }, "keys": "ctrl+alt+," },
         { "command": "find", "keys": "ctrl+shift+f" },
-        { "command": "toggleRetroEffect" },
         { "command": "openTabColorPicker" },
         { "command": "commandPalette", "keys":"ctrl+shift+p" },
 

src/cascadia/TerminalSettingsModel/defaults.json

@@ -285,7 +285,7 @@
         { "command": "openSettings", "keys": "ctrl+," },
         { "command": { "action": "openSettings", "target": "defaultsFile" }, "keys": "ctrl+alt+," },
         { "command": "find", "keys": "ctrl+shift+f" },
-        { "command": "toggleRetroEffect" },
+        { "command": "toggleShaderEffects" },
         { "command": "openTabColorPicker" },
         { "command": "renameTab" },
         { "command": "openTabRenamer" },

src/renderer/dx/DxRenderer.cpp

@@ -87,7 +87,9 @@ DxEngine::DxEngine() :
     _swapChainDesc{ 0 },
     _swapChainFrameLatencyWaitableObject{ INVALID_HANDLE_VALUE },
     _recreateDeviceRequested{ false },
-    _retroTerminalEffects{ false },
+    _terminalEffectsEnabled{ false },
+    _retroTerminalEffect{ false },
+    _pixelShaderPath{},
     _forceFullRepaintRendering{ false },
     _softwareRendering{ false },
     _antialiasingMode{ D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE },
@@ -229,6 +231,85 @@ _CompileShader(
 #endif
 }
 
+// Routine Description:
+// - Checks if terminal effects are enabled.
+// Arguments:
+// Return Value:
+// - True if terminal effects are enabled
+bool DxEngine::_HasTerminalEffects() const noexcept
+{
+    return _terminalEffectsEnabled && (_retroTerminalEffect || !_pixelShaderPath.empty());
+}
+
+// Routine Description:
+// - Toggles terminal effects off and on. If no terminal effect is configured has no effect
+// Arguments:
+// Return Value:
+// - Void
+void DxEngine::ToggleShaderEffects()
+{
+    _terminalEffectsEnabled = !_terminalEffectsEnabled;
+    LOG_IF_FAILED(InvalidateAll());
+}
+
+// Routine Description:
+// - Loads pixel shader source depending on _retroTerminalEffect and _pixelShaderPath
+// Arguments:
+// Return Value:
+// - Pixel shader source code
+std::string DxEngine::_LoadPixelShaderFile() const
+{
+    // If the user specified the new pixel shader, it has precedence
+    if (!_pixelShaderPath.empty())
+    {
+        try
+        {
+            wil::unique_hfile hFile{ CreateFileW(_pixelShaderPath.c_str(),
+                                                 GENERIC_READ,
+                                                 FILE_SHARE_READ,
+                                                 nullptr,
+                                                 OPEN_EXISTING,
+                                                 FILE_ATTRIBUTE_NORMAL,
+                                                 nullptr) };
+
+            THROW_LAST_ERROR_IF(!hFile); // This will be caught below.
+
+            // fileSize is in bytes
+            const auto fileSize = GetFileSize(hFile.get(), nullptr);
+            THROW_LAST_ERROR_IF(fileSize == INVALID_FILE_SIZE);
+
+            std::vector<char> utf8buffer;
+            utf8buffer.reserve(fileSize);
+
+            DWORD bytesRead = 0;
+            THROW_LAST_ERROR_IF(!ReadFile(hFile.get(), utf8buffer.data(), fileSize, &bytesRead, nullptr));
+
+            // convert buffer to UTF-8 string
+            std::string utf8string(utf8buffer.data(), fileSize);
+
+            return utf8string;
+        }
+        catch (...)
+        {
+            // If we ran into any problems during loading pixel shader, call to
+            // the warning callback to surface the file not found error
+            const auto exceptionHr = LOG_CAUGHT_EXCEPTION();
+            if (_pfnWarningCallback)
+            {
+                _pfnWarningCallback(exceptionHr);
+            }
+
+            return std::string{};
+        }
+    }
+    else if (_retroTerminalEffect)
+    {
+        return std::string{ retroPixelShaderString };
+    }
+
+    return std::string{};
+}
+
 // Routine Description:
 // - Setup D3D objects for doing shader things for terminal effects.
 // Arguments:
@@ -236,6 +317,19 @@ _CompileShader(
 // - HRESULT status.
 HRESULT DxEngine::_SetupTerminalEffects()
 {
+    _pixelShaderLoaded = false;
+
+    const auto pixelShaderSource = _LoadPixelShaderFile();
+    if (pixelShaderSource.empty())
+    {
+        // There's no shader to compile. This might be due to failing to load,
+        // or because there's just no shader enabled at all.
+        // Turn the effects off for now.
+        _terminalEffectsEnabled = false;
+
+        return S_FALSE;
+    }
+
     ::Microsoft::WRL::ComPtr<ID3D11Texture2D> swapBuffer;
     RETURN_IF_FAILED(_dxgiSwapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), (LPVOID*)&swapBuffer));
 
@@ -260,12 +354,26 @@ HRESULT DxEngine::_SetupTerminalEffects()
 
     // Prepare shaders.
     auto vertexBlob = _CompileShader(screenVertexShaderString, "vs_5_0");
-    auto pixelBlob = _CompileShader(screenPixelShaderString, "ps_5_0");
+    Microsoft::WRL::ComPtr<ID3DBlob> pixelBlob;
-    // TODO:GH#3928 move the shader files to to hlsl files and package their
+    // As the pixel shader source is user provided it's possible there's a problem with it
-    // build output to UWP app and load with these.
+    //  so load it inside a try catch, on any error log and fallback on the error pixel shader
-    // ::Microsoft::WRL::ComPtr<ID3DBlob> vertexBlob, pixelBlob;
+    //  If even the error pixel shader fails to load rely on standard exception handling
-    // RETURN_IF_FAILED(D3DReadFileToBlob(L"ScreenVertexShader.cso", &vertexBlob));
+    try
-    // RETURN_IF_FAILED(D3DReadFileToBlob(L"ScreenPixelShader.cso", &pixelBlob));
+    {
+        pixelBlob = _CompileShader(pixelShaderSource, "ps_5_0");
+    }
+    catch (...)
+    {
+        // Call to the warning callback to surface the shader compile error
+        const auto exceptionHr = LOG_CAUGHT_EXCEPTION();
+        if (_pfnWarningCallback)
+        {
+            // If this fails, it'll return E_FAIL, which is terribly
+            // uninformative. Instead, raise something more useful.
+            _pfnWarningCallback(D2DERR_SHADER_COMPILE_FAILED);
+        }
+        return exceptionHr;
+    }
 
     RETURN_IF_FAILED(_d3dDevice->CreateVertexShader(
         vertexBlob->GetBufferPointer(),
@@ -329,23 +437,47 @@ HRESULT DxEngine::_SetupTerminalEffects()
     // Create the texture sampler state.
     RETURN_IF_FAILED(_d3dDevice->CreateSamplerState(&samplerDesc, &_samplerState));
 
+    _pixelShaderLoaded = true;
     return S_OK;
 }
 
 // Routine Description:
 // - Puts the correct values in _pixelShaderSettings, so the struct can be
-//   passed the GPU.
+//   passed the GPU and updates the GPU resource.
 // Arguments:
 // - <none>
 // Return Value:
 // - <none>
 void DxEngine::_ComputePixelShaderSettings() noexcept
 {
-    // Retro scan lines alternate every pixel row at 100% scaling.
+    if (_HasTerminalEffects() && _d3dDeviceContext && _pixelShaderSettingsBuffer)
-    _pixelShaderSettings.ScaledScanLinePeriod = _scale * 1.0f;
+    {
+        try
+        {
+            // Set the time
+            //  TODO:GH#7013 Grab timestamp
+            _pixelShaderSettings.Time = 0.0f;
 
-    // Gaussian distribution sigma used for blurring.
+            // Set the UI Scale
-    _pixelShaderSettings.ScaledGaussianSigma = _scale * 2.0f;
+            _pixelShaderSettings.Scale = _scale;
+
+            // Set the display resolution
+            const float w = 1.0f * _displaySizePixels.width<UINT>();
+            const float h = 1.0f * _displaySizePixels.height<UINT>();
+            _pixelShaderSettings.Resolution = XMFLOAT2{ w, h };
+
+            // Set the background
+            DirectX::XMFLOAT4 background{};
+            background.x = _backgroundColor.r;
+            background.y = _backgroundColor.g;
+            background.z = _backgroundColor.b;
+            background.w = _backgroundColor.a;
+            _pixelShaderSettings.Background = background;
+
+            _d3dDeviceContext->UpdateSubresource(_pixelShaderSettingsBuffer.Get(), 0, nullptr, &_pixelShaderSettings, 0, 0);
+        }
+        CATCH_LOG();
+    }
 }
 
 // Routine Description;
@@ -521,13 +653,13 @@ try
             }
         }
 
-        if (_retroTerminalEffects)
+        if (_HasTerminalEffects())
         {
             const HRESULT hr = _SetupTerminalEffects();
             if (FAILED(hr))
             {
-                _retroTerminalEffects = false;
                 LOG_HR_MSG(hr, "Failed to setup terminal effects. Disabling.");
+                _terminalEffectsEnabled = false;
             }
         }
 
@@ -560,6 +692,8 @@ try
         CATCH_LOG(); // A failure in the notification function isn't a failure to prepare, so just log it and go on.
     }
 
+    _recreateDeviceRequested = false;
+
     return S_OK;
 }
 CATCH_RETURN();
@@ -675,7 +809,15 @@ void DxEngine::_ReleaseDeviceResources() noexcept
     {
         _haveDeviceResources = false;
 
+        // Destroy Terminal Effect resources
+        _renderTargetView.Reset();
+        _vertexShader.Reset();
+        _pixelShader.Reset();
+        _vertexLayout.Reset();
+        _screenQuadVertexBuffer.Reset();
         _pixelShaderSettingsBuffer.Reset();
+        _samplerState.Reset();
+        _framebufferCapture.Reset();
 
         _d2dBrushForeground.Reset();
         _d2dBrushBackground.Reset();
@@ -802,17 +944,38 @@ void DxEngine::SetCallback(std::function<void()> pfn)
     _pfn = pfn;
 }
 
-bool DxEngine::GetRetroTerminalEffects() const noexcept
+void DxEngine::SetWarningCallback(std::function<void(const HRESULT)> pfn)
 {
-    return _retroTerminalEffects;
+    _pfnWarningCallback = pfn;
 }
 
-void DxEngine::SetRetroTerminalEffects(bool enable) noexcept
+bool DxEngine::GetRetroTerminalEffect() const noexcept
+{
+    return _retroTerminalEffect;
+}
+
+void DxEngine::SetRetroTerminalEffect(bool enable) noexcept
 try
 {
-    if (_retroTerminalEffects != enable)
+    if (_retroTerminalEffect != enable)
     {
-        _retroTerminalEffects = enable;
+        // Enable shader effects if the path isn't empty. Otherwise leave it untouched.
+        _terminalEffectsEnabled = enable ? true : _terminalEffectsEnabled;
+        _retroTerminalEffect = enable;
+        _recreateDeviceRequested = true;
+        LOG_IF_FAILED(InvalidateAll());
+    }
+}
+CATCH_LOG()
+
+void DxEngine::SetPixelShaderPath(std::wstring_view value) noexcept
+try
+{
+    if (_pixelShaderPath != value)
+    {
+        // Enable shader effects if the path isn't empty. Otherwise leave it untouched.
+        _terminalEffectsEnabled = value.empty() ? _terminalEffectsEnabled : true;
+        _pixelShaderPath = { value };
         _recreateDeviceRequested = true;
         LOG_IF_FAILED(InvalidateAll());
     }
@@ -1085,14 +1248,9 @@ try
 {
     RETURN_HR_IF(E_NOT_VALID_STATE, _isPainting); // invalid to start a paint while painting.
 
-    // If someone explicitly requested differential rendering off, then we need to invalidate everything
+    // If full repaints are needed then we need to invalidate everything
     // so the entire frame is repainted.
-    //
+    if (_FullRepaintNeeded())
-    // If retro terminal effects are on, we must invalidate everything for them to draw correctly.
-    // Yes, this will further impact the performance of retro terminal effects.
-    // But we're talking about running the entire display pipeline through a shader for
-    // cosmetic effect, so performance isn't likely the top concern with this feature.
-    if (_forceFullRepaintRendering || _retroTerminalEffects)
     {
         _invalidMap.set_all();
     }
@@ -1118,7 +1276,6 @@ try
         if (!_haveDeviceResources || _recreateDeviceRequested)
         {
             RETURN_IF_FAILED(_CreateDeviceResources(true));
-            _recreateDeviceRequested = false;
         }
         else if (_displaySizePixels != clientSize || _prevScale != _scale)
         {
@@ -1312,12 +1469,12 @@ void DxEngine::WaitUntilCanRender() noexcept
 {
     if (_presentReady)
     {
-        if (_retroTerminalEffects)
+        if (_HasTerminalEffects() && _pixelShaderLoaded)
         {
             const HRESULT hr2 = _PaintTerminalEffects();
             if (FAILED(hr2))
             {
-                _retroTerminalEffects = false;
+                _pixelShaderLoaded = false;
                 LOG_HR_MSG(hr2, "Failed to paint terminal effects. Disabling.");
             }
         }
@@ -1378,10 +1535,15 @@ void DxEngine::WaitUntilCanRender() noexcept
                 }
             }
 
-            // Finally copy the front image (being presented now) onto the backing buffer
+            // If we are doing full repaints we don't need to copy front buffer to back buffer
-            // (where we are about to draw the next frame) so we can draw only the differences
+            if (!_FullRepaintNeeded())
-            // next frame.
+            {
-            RETURN_IF_FAILED(_CopyFrontToBack());
+                // Finally copy the front image (being presented now) onto the backing buffer
+                // (where we are about to draw the next frame) so we can draw only the differences
+                // next frame.
+                RETURN_IF_FAILED(_CopyFrontToBack());
+            }
+
             _presentReady = false;
 
             _presentDirty.clear();
@@ -1687,6 +1849,18 @@ try
 }
 CATCH_RETURN()
 
+[[nodiscard]] bool DxEngine::_FullRepaintNeeded() const noexcept
+{
+    // If someone explicitly requested differential rendering off, then we need to invalidate everything
+    // so the entire frame is repainted.
+    //
+    // If terminal effects are on, we must invalidate everything for them to draw correctly.
+    // Yes, this will further impact the performance of terminal effects.
+    // But we're talking about running the entire display pipeline through a shader for
+    // cosmetic effect, so performance isn't likely the top concern with this feature.
+    return _forceFullRepaintRendering || _HasTerminalEffects();
+}
+
 // Routine Description:
 // - Updates the default brush colors used for drawing
 // Arguments:
@@ -1749,6 +1923,9 @@ CATCH_RETURN()
         _hyperlinkStrokeStyle = (textAttributes.GetHyperlinkId() == _hyperlinkHoveredId) ? _strokeStyle : _dashStrokeStyle;
     }
 
+    // Update pixel shader settings as background color might have changed
+    _ComputePixelShaderSettings();
+
     return S_OK;
 }
 
@@ -1813,15 +1990,8 @@ CATCH_RETURN();
 
     RETURN_IF_FAILED(InvalidateAll());
 
-    if (_retroTerminalEffects && _d3dDeviceContext && _pixelShaderSettingsBuffer)
+    // Update pixel shader settings as scale might have changed
-    {
+    _ComputePixelShaderSettings();
-        _ComputePixelShaderSettings();
-        try
-        {
-            _d3dDeviceContext->UpdateSubresource(_pixelShaderSettingsBuffer.Get(), 0, nullptr, &_pixelShaderSettings, 0, 0);
-        }
-        CATCH_RETURN();
-    }
 
     return S_OK;
 }

src/renderer/dx/DxRenderer.hpp

@@ -15,6 +15,7 @@
 #include <d2d1.h>
 #include <d2d1_1.h>
 #include <d2d1helper.h>
+#include <DirectXMath.h>
 #include <dwrite.h>
 #include <dwrite_1.h>
 #include <dwrite_2.h>
@@ -55,9 +56,14 @@ namespace Microsoft::Console::Render
         [[nodiscard]] HRESULT SetWindowSize(const SIZE pixels) noexcept;
 
         void SetCallback(std::function<void()> pfn);
+        void SetWarningCallback(std::function<void(const HRESULT)> pfn);
 
-        bool GetRetroTerminalEffects() const noexcept;
+        void ToggleShaderEffects();
-        void SetRetroTerminalEffects(bool enable) noexcept;
+
+        bool GetRetroTerminalEffect() const noexcept;
+        void SetRetroTerminalEffect(bool enable) noexcept;
+
+        void SetPixelShaderPath(std::wstring_view value) noexcept;
 
         void SetForceFullRepaintRendering(bool enable) noexcept;
 
@@ -124,6 +130,7 @@ namespace Microsoft::Console::Render
     protected:
         [[nodiscard]] HRESULT _DoUpdateTitle(_In_ const std::wstring& newTitle) noexcept override;
         [[nodiscard]] HRESULT _PaintTerminalEffects() noexcept;
+        [[nodiscard]] bool _FullRepaintNeeded() const noexcept;
 
     private:
         enum class SwapChainMode
@@ -141,6 +148,7 @@ namespace Microsoft::Console::Render
         float _prevScale;
 
         std::function<void()> _pfn;
+        std::function<void(const HRESULT)> _pfnWarningCallback;
 
         bool _isEnabled;
         bool _isPainting;
@@ -221,7 +229,22 @@ namespace Microsoft::Console::Render
         std::unique_ptr<DrawingContext> _drawingContext;
 
         // Terminal effects resources.
-        bool _retroTerminalEffects;
+
+        // Controls if configured terminal effects are enabled
+        bool _terminalEffectsEnabled;
+
+        // Experimental and deprecated retro terminal effect
+        //  Preserved for backwards compatibility
+        //  Implemented in terms of the more generic pixel shader effect
+        //  Has precendence over pixel shader effect
+        bool _retroTerminalEffect;
+
+        // Experimental and pixel shader effect
+        //  Allows user to load a pixel shader from a few presets or from a file path
+        std::wstring _pixelShaderPath;
+        bool _pixelShaderLoaded{ false };
+
+        // DX resources needed for terminal effects
         ::Microsoft::WRL::ComPtr<ID3D11RenderTargetView> _renderTargetView;
         ::Microsoft::WRL::ComPtr<ID3D11VertexShader> _vertexShader;
         ::Microsoft::WRL::ComPtr<ID3D11PixelShader> _pixelShader;
@@ -242,12 +265,19 @@ namespace Microsoft::Console::Render
         // DirectX constant buffers need to be a multiple of 16; align to pad the size.
         __declspec(align(16)) struct
         {
-            float ScaledScanLinePeriod;
+            // Note: This can be seen as API endpoint towards user provided pixel shaders.
-            float ScaledGaussianSigma;
+            //  Changes here can break existing pixel shaders so be careful with changing datatypes
+            //  and order of parameters
+            float Time;
+            float Scale;
+            DirectX::XMFLOAT2 Resolution;
+            DirectX::XMFLOAT4 Background;
 #pragma warning(suppress : 4324) // structure was padded due to __declspec(align())
         } _pixelShaderSettings;
 
         [[nodiscard]] HRESULT _CreateDeviceResources(const bool createSwapChain) noexcept;
+        bool _HasTerminalEffects() const noexcept;
+        std::string _LoadPixelShaderFile() const;
         HRESULT _SetupTerminalEffects();
         void _ComputePixelShaderSettings() noexcept;
 

src/renderer/dx/ScreenPixelShader.h

@@ -1,19 +1,23 @@
 #pragma once
 
 #ifdef __INSIDE_WINDOWS
-const char screenPixelShaderString[] = "";
+constexpr std::string_view retroPixelShaderString{ "" };
 #else
-const char screenPixelShaderString[] = R"(
+constexpr std::string_view retroPixelShaderString{ R"(
+// The original retro pixel shader
 Texture2D shaderTexture;
 SamplerState samplerState;
 
-cbuffer PixelShaderSettings
+cbuffer PixelShaderSettings {
-{
+  float  Time;
-    float ScaledScanLinePeriod;
+  float  Scale;
-    float ScaledGaussianSigma;
+  float2 Resolution;
+  float4 Background;
 };
 
 #define SCANLINE_FACTOR 0.5
+#define SCALED_SCANLINE_PERIOD Scale
+#define SCALED_GAUSSIAN_SIGMA (2.0*Scale)
 
 static const float M_PI = 3.14159265f;
 
@@ -53,7 +57,7 @@ float4 Blur(Texture2D input, float2 tex_coord, float sigma)
 
 float SquareWave(float y)
 {
-    return 1 - (floor(y / ScaledScanLinePeriod) % 2) * SCANLINE_FACTOR;
+    return 1 - (floor(y / SCALED_SCANLINE_PERIOD) % 2) * SCANLINE_FACTOR;
 }
 
 float4 Scanline(float4 color, float4 pos)
@@ -78,10 +82,10 @@ float4 main(float4 pos : SV_POSITION, float2 tex : TEXCOORD) : SV_TARGET
 
     // TODO:GH#3930 Make these configurable in some way.
     float4 color = input.Sample(samplerState, tex);
-    color += Blur(input, tex, ScaledGaussianSigma)*0.3;
+    color += Blur(input, tex, SCALED_GAUSSIAN_SIGMA)*0.3;
     color = Scanline(color, pos);
 
     return color;
 }
-)";
+)" };
 #endif