It can actually load and link something

- Lots of refactoring in the shader loading code.
 - Abstract all the different shader source transformations into ProcessingStage.
 - An ordered list of ProcessingStages is called a ShaderTransformer.
 - When you acquire sources, a Shader now keeps track of the source location, shader type, and source code all together.
This commit is contained in:
JozsefA 2021-05-12 15:14:33 -07:00
parent c1b7a1c19d
commit 56b2046957
23 changed files with 290 additions and 114 deletions

View file

@ -7,7 +7,8 @@ import com.jozufozu.flywheel.backend.gl.shader.GlProgram;
import com.jozufozu.flywheel.backend.gl.shader.IMultiProgram;
import com.jozufozu.flywheel.backend.gl.shader.ProgramSpec;
import com.jozufozu.flywheel.backend.gl.shader.ShaderSpecLoader;
import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
import com.jozufozu.flywheel.backend.loading.ProcessingStage;
import com.jozufozu.flywheel.backend.loading.Shader;
import net.minecraft.util.ResourceLocation;
@ -32,8 +33,11 @@ public abstract class ShaderContext<P extends GlProgram> {
Backend.log.debug("Loaded program {}", programSpec.name);
}
public String preProcess(ShaderLoader loader, ShaderType type, ResourceLocation shader, String shaderSrc) {
return shaderSrc;
public void preProcess(ShaderLoader loader, Shader shader) {
}
public ProcessingStage loadingStage(ShaderLoader loader) {
return shader -> this.preProcess(loader, shader);
}
public P getProgram(ProgramSpec spec) {

View file

@ -14,6 +14,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
@ -25,12 +26,12 @@ import java.util.stream.Stream;
import org.lwjgl.system.MemoryUtil;
import com.google.common.collect.Lists;
import com.jozufozu.flywheel.backend.gl.attrib.IVertexAttrib;
import com.jozufozu.flywheel.backend.gl.GlObject;
import com.jozufozu.flywheel.backend.gl.shader.GlProgram;
import com.jozufozu.flywheel.backend.gl.shader.GlShader;
import com.jozufozu.flywheel.backend.gl.shader.ProgramSpec;
import com.jozufozu.flywheel.backend.gl.shader.ShaderConstants;
import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
import com.jozufozu.flywheel.backend.loading.Shader;
import com.jozufozu.flywheel.backend.loading.ShaderTransformer;
import com.mojang.blaze3d.systems.RenderSystem;
import net.minecraft.resources.IResource;
@ -58,15 +59,24 @@ public class ShaderLoader {
shaderSource.clear();
loadShaderSources(manager);
// ResourceLocation test = new ResourceLocation("create", "model_new.vert");
// ResourceLocation vert = new ResourceLocation("create", "skeleton/instanced/instanced.vert");
// InstancedArraysTemplate template = new InstancedArraysTemplate(this);
//
// InstancedArraysShaderTemplate template = new InstancedArraysShaderTemplate(getShaderSource(vert));
// ParsedShader parsedShader = new ParsedShader(getShaderSource(test));
// ResourceLocation name = new ResourceLocation("create", "test");
// ResourceLocation vert = new ResourceLocation("create", "model_new.vert");
// ResourceLocation frag = new ResourceLocation("create", "block_new.frag");
//
// String apply = template.apply(parsedShader);
// ShaderTransformer transformer = new ShaderTransformer()
// .pushStage(WorldContext.INSTANCE.loadingStage(this))
// .pushStage(this::processIncludes)
// .pushStage(template)
// .pushStage(this::processIncludes);
//
// printSource(test, apply);
// Shader vertexFile = this.source(vert, ShaderType.VERTEX);
// Shader fragmentFile = this.source(frag, ShaderType.FRAGMENT);
//
// GlProgram.Builder builder = loadProgram(name, transformer, vertexFile, fragmentFile);
//
// BasicProgram program = new BasicProgram(builder, GlFogMode.NONE.getFogFactory());
for (ShaderContext<?> context : Backend.contexts.values()) {
context.load(this);
@ -108,48 +118,41 @@ public class ShaderLoader {
}
}
public GlProgram.Builder loadProgram(ShaderContext<?> ctx, ProgramSpec programSpec) {
return loadProgram(ctx, programSpec, programSpec.defines);
public Shader source(ResourceLocation name, ShaderType type) {
return new Shader(type, name, getShaderSource(name));
}
public GlProgram.Builder loadProgram(ShaderContext<?> ctx, ProgramSpec programSpec, ShaderConstants defines) {
return loadProgram(ctx, programSpec.name, programSpec.vert, programSpec.frag, programSpec.attributes, defines);
public GlProgram.Builder loadProgram(ResourceLocation name, ShaderTransformer transformer, Shader... shaders) {
return loadProgram(name, transformer, Lists.newArrayList(shaders));
}
public GlProgram.Builder loadProgram(ShaderContext<?> ctx, ResourceLocation name, ResourceLocation vert, ResourceLocation frag, Collection<IVertexAttrib> attribs, ShaderConstants defines) {
GlShader vsh = null;
GlShader fsh = null;
/**
* Ingests the given shaders, compiling them and linking them together after applying the transformer to the source.
*
* @param name What should we call this program if something goes wrong?
* @param transformer What should we do to the sources before compilation?
* @param shaders What are the different shader stages that should be linked together?
* @return A linked program builder.
*/
public GlProgram.Builder loadProgram(ResourceLocation name, ShaderTransformer transformer, Collection<Shader> shaders) {
List<GlShader> compiled = new ArrayList<>(shaders.size());
try {
vsh = loadShader(ctx, vert, ShaderType.VERTEX, defines);
fsh = loadShader(ctx, frag, ShaderType.FRAGMENT, defines);
GlProgram.Builder builder = GlProgram.builder(name);
return GlProgram.builder(name)
.attachShader(vsh)
.attachShader(fsh)
.addAttributes(attribs)
.link();
for (Shader shader : shaders) {
transformer.transformSource(shader);
GlShader sh = new GlShader(shader);
compiled.add(sh);
builder.attachShader(sh);
}
return builder.link();
} finally {
if (vsh != null) vsh.delete();
if (fsh != null) fsh.delete();
compiled.forEach(GlObject::delete);
}
}
public GlShader loadShader(ShaderContext<?> ctx, ResourceLocation name, ShaderType type, ShaderConstants defines) {
String source = shaderSource.get(name);
source = ctx.preProcess(this, type, name, source);
source = processIncludes(source, name);
if (defines != null)
source = defines.process(source);
if (debugDumpFile) {
printSource(name, source);
}
return new GlShader(type, name, source);
}
private void printSource(ResourceLocation name, String source) {
Backend.log.debug("Finished processing '" + name + "':");
int i = 1;
@ -158,11 +161,12 @@ public class ShaderLoader {
}
}
private String processIncludes(String source, ResourceLocation baseName) {
public void processIncludes(Shader shader) {
HashSet<ResourceLocation> seen = new HashSet<>();
seen.add(baseName);
seen.add(shader.name);
return includeRecursive(source, seen).collect(Collectors.joining("\n"));
String includesInjected = includeRecursive(shader.getSource(), seen).collect(Collectors.joining("\n"));
shader.setSource(includesInjected);
}
private Stream<String> includeRecursive(String source, Set<ResourceLocation> seen) {

View file

@ -22,6 +22,10 @@ public class BasicProgram extends GlProgram {
protected int uBlockAtlas;
protected int uLightMap;
public BasicProgram(GlProgram.Builder builder, ProgramFogMode.Factory fogFactory) {
this(builder.name, builder.program, fogFactory);
}
public BasicProgram(ResourceLocation name, int handle, ProgramFogMode.Factory fogFactory) {
super(name, handle);
uTime = getUniformLocation("uTime");

View file

@ -14,6 +14,7 @@ import com.jozufozu.flywheel.backend.gl.shader.IMultiProgram;
import com.jozufozu.flywheel.backend.gl.shader.ShaderSpecLoader;
import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
import com.jozufozu.flywheel.backend.instancing.MaterialSpec;
import com.jozufozu.flywheel.backend.loading.Shader;
import net.minecraft.util.ResourceLocation;
@ -50,15 +51,15 @@ public class WorldContext<P extends BasicProgram> extends ShaderContext<P> {
}
@Override
public String preProcess(ShaderLoader loader, ShaderType type, ResourceLocation shader, String shaderSrc) {
String builtinSrc = loader.getShaderSource(builtins.get(type));
public void preProcess(ShaderLoader loader, Shader shader) {
String builtinSrc = loader.getShaderSource(builtins.get(shader.type));
Matcher matcher = builtinPattern.matcher(shaderSrc);
Matcher matcher = builtinPattern.matcher(shader.getSource());
if (matcher.find())
return matcher.replaceFirst(builtinSrc);
throw new RuntimeException(String.format("%s shader '%s' is missing %s, cannot use in World Context", type.name, shader, declaration));
shader.setSource(matcher.replaceFirst(builtinSrc));
else
throw new RuntimeException(String.format("%s shader '%s' is missing %s, cannot use in World Context", shader.type.name, shader.name, declaration));
}
@Override

View file

@ -45,7 +45,7 @@ public class FogSensitiveProgram<P extends GlProgram> implements IMultiProgram<P
defines.defineAll(fogMode.getDefines());
GlProgram.Builder builder = loader.loadProgram(ctx, spec, defines);
GlProgram.Builder builder = spec.loadProgram(ctx, defines, loader);
programs.put(fogMode, fogProgramLoader.create(builder.name, builder.program, fogMode.getFogFactory()));
}

View file

@ -5,6 +5,7 @@ import org.lwjgl.opengl.GL20;
import com.jozufozu.flywheel.backend.Backend;
import com.jozufozu.flywheel.backend.gl.GlObject;
import com.jozufozu.flywheel.backend.gl.versioned.GlCompat;
import com.jozufozu.flywheel.backend.loading.Shader;
import net.minecraft.util.ResourceLocation;
@ -13,6 +14,10 @@ public class GlShader extends GlObject {
public final ResourceLocation name;
public final ShaderType type;
public GlShader(Shader shader) {
this(shader.type, shader.name, shader.getSource());
}
public GlShader(ShaderType type, ResourceLocation name, String source) {
this.type = type;
this.name = name;
@ -38,13 +43,4 @@ public class GlShader extends GlObject {
protected void deleteInternal(int handle) {
GL20.glDeleteShader(handle);
}
@FunctionalInterface
public interface PreProcessor {
String process(String source);
default PreProcessor andThen(PreProcessor that) {
return source -> that.process(this.process(source));
}
}
}

View file

@ -3,7 +3,12 @@ package com.jozufozu.flywheel.backend.gl.shader;
import java.util.ArrayList;
import java.util.Arrays;
import com.jozufozu.flywheel.backend.ShaderContext;
import com.jozufozu.flywheel.backend.ShaderLoader;
import com.jozufozu.flywheel.backend.gl.attrib.IVertexAttrib;
import com.jozufozu.flywheel.backend.loading.InstancedArraysTemplate;
import com.jozufozu.flywheel.backend.loading.Shader;
import com.jozufozu.flywheel.backend.loading.ShaderTransformer;
import net.minecraft.util.ResourceLocation;
@ -30,6 +35,24 @@ public class ProgramSpec {
this.attributes = attributes;
}
public GlProgram.Builder loadProgram(ShaderContext<?> ctx, ShaderConstants defines, ShaderLoader loader) {
InstancedArraysTemplate template = new InstancedArraysTemplate(loader);
ShaderTransformer transformer = new ShaderTransformer()
.pushStage(ctx.loadingStage(loader))
// .pushStage(loader::processIncludes)
// .pushStage(template)
.pushStage(loader::processIncludes);
if (defines != null)
transformer.pushStage(defines);
Shader vertexFile = loader.source(vert, ShaderType.VERTEX);
Shader fragmentFile = loader.source(frag, ShaderType.FRAGMENT);
return loader.loadProgram(name, transformer, vertexFile, fragmentFile)
.addAttributes(attributes);
}
public static class Builder {
private ResourceLocation vert;
private ResourceLocation frag;

View file

@ -8,8 +8,10 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.collect.Lists;
import com.jozufozu.flywheel.backend.loading.ProcessingStage;
import com.jozufozu.flywheel.backend.loading.Shader;
public class ShaderConstants implements GlShader.PreProcessor {
public class ShaderConstants implements ProcessingStage {
public static final ShaderConstants EMPTY = new ShaderConstants();
private final ArrayList<String> defines;
@ -45,8 +47,8 @@ public class ShaderConstants implements GlShader.PreProcessor {
}
@Override
public String process(String source) {
return new BufferedReader(new StringReader(source)).lines().flatMap(line -> {
public void process(Shader shader) {
shader.setSource(new BufferedReader(new StringReader(shader.getSource())).lines().flatMap(line -> {
Stream<String> map = Stream.of(line);
if (line.startsWith("#version")) {
@ -54,6 +56,6 @@ public class ShaderConstants implements GlShader.PreProcessor {
}
return map;
}).collect(Collectors.joining("\n"));
}).collect(Collectors.joining("\n")));
}
}

View file

@ -31,7 +31,7 @@ public class SingleProgram<P extends GlProgram> implements IMultiProgram<P> {
@Override
public IMultiProgram<P> create(ShaderLoader loader, ShaderContext<P> ctx, ProgramSpec spec) {
GlProgram.Builder builder = loader.loadProgram(ctx, spec);
GlProgram.Builder builder = spec.loadProgram(ctx, spec.defines, loader);
return new SingleProgram<>(factory.create(builder.name, builder.program));
}

View file

@ -0,0 +1,21 @@
package com.jozufozu.flywheel.backend.loading;
import com.jozufozu.flywheel.backend.ShaderLoader;
import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
import net.minecraft.util.ResourceLocation;
public class InstancedArraysTemplate extends ProgramTemplate {
public static final String[] requiredVert = {"FLWInstanceData", "FLWVertexData", "FLWFragment"};
public static final String[] requiredFrag = {"FLWFragment"};
public static final ResourceLocation vert = new ResourceLocation("create", "template/instanced/instanced.vert");
public static final ResourceLocation frag = new ResourceLocation("create", "template/instanced/instanced.frag");
public InstancedArraysTemplate(ShaderLoader loader) {
super(loader);
templates.put(ShaderType.VERTEX, new ShaderTemplate(requiredVert, loader.getShaderSource(vert)));
templates.put(ShaderType.FRAGMENT, new ShaderTemplate(requiredFrag, loader.getShaderSource(frag)));
}
}

View file

@ -5,17 +5,20 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.minecraft.util.ResourceLocation;
public class ParsedShader {
private static final Pattern decorator = Pattern.compile("#\\[([\\w_]*)]");
private static final Pattern taggedStruct = Pattern.compile("#\\[([\\w_]*)]\\s*struct\\s+([\\w\\d_]*)\\s*\\{(\\s*(?:.*;\\s*\\n)+\\s*)}\\s*;");
final ResourceLocation loc;
final String src;
final Map<String, TaggedStruct> tag2Struct = new HashMap<>();
final Map<String, TaggedStruct> name2Struct = new HashMap<>();
public ParsedShader(String src) {
public ParsedShader(ResourceLocation loc, String src) {
this.loc = loc;
Matcher structs = taggedStruct.matcher(src);
StringBuffer strippedSrc = new StringBuffer();

View file

@ -0,0 +1,7 @@
package com.jozufozu.flywheel.backend.loading;
@FunctionalInterface
public interface ProcessingStage {
void process(Shader shader);
}

View file

@ -0,0 +1,28 @@
package com.jozufozu.flywheel.backend.loading;
import java.util.EnumMap;
import java.util.Map;
import com.jozufozu.flywheel.backend.ShaderLoader;
import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
public class ProgramTemplate implements ProcessingStage {
protected final ShaderLoader loader;
protected Map<ShaderType, ShaderTemplate> templates = new EnumMap<>(ShaderType.class);
public ProgramTemplate(ShaderLoader loader) {
this.loader = loader;
}
@Override
public void process(Shader shader) {
ShaderTemplate template = templates.get(shader.type);
if (template == null) return;
ParsedShader parsedShader = new ParsedShader(shader.name, shader.getSource());
shader.setSource(template.apply(parsedShader));
}
}

View file

@ -0,0 +1,33 @@
package com.jozufozu.flywheel.backend.loading;
import com.jozufozu.flywheel.backend.gl.shader.ShaderType;
import net.minecraft.util.ResourceLocation;
public class Shader {
public ShaderType type;
public ResourceLocation name;
private String source;
public Shader(ShaderType type, ResourceLocation name, String source) {
this.type = type;
this.name = name;
this.setSource(source);
}
public static Shader vert(ResourceLocation fileLoc, String source) {
return new Shader(ShaderType.VERTEX, fileLoc, source);
}
public static Shader frag(ResourceLocation fileLoc, String source) {
return new Shader(ShaderType.FRAGMENT, fileLoc, source);
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
}

View file

@ -1,22 +1,26 @@
package com.jozufozu.flywheel.backend.loading;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class InstancedArraysShaderTemplate {
public class ShaderTemplate {
private static final String delimiter = "#flwbeginbody";
private static final Pattern headerFinder = Pattern.compile(delimiter);
private static final Pattern prefixer = Pattern.compile("#FLWPrefixFields\\((\\S+),\\s*([^\\n]+)\\)");
private static final Pattern prefixer = Pattern.compile("#FLWPrefixFields\\((\\w+),\\s*(\\w+),\\s*([\\w\\d]+)\\)");
private static final Pattern assigner = Pattern.compile("#FLWAssignFields\\(([\\w\\d_]+),\\s*([\\w\\d_.]+),\\s*([\\w\\d_.]+)\\)");
public static final String[] required = {"FLWInstanceData", "FLWVertexData", "FLWFragment"};
final String[] requiredStructs;
final String header;
final String body;
public InstancedArraysShaderTemplate(String templateSrc) {
public ShaderTemplate(String[] requiredStructs, String templateSrc) {
this.requiredStructs = requiredStructs;
Matcher matcher = headerFinder.matcher(templateSrc);
if (!matcher.find()) {
@ -38,10 +42,21 @@ public class InstancedArraysShaderTemplate {
public String processBody(ParsedShader shader) {
String s = body;
for (String name : required) {
List<String> missing = new ArrayList<>();
for (String name : requiredStructs) {
TaggedStruct struct = shader.getTag(name);
if (struct != null) {
s = s.replace(name, struct.name);
} else {
missing.add(name);
}
}
if (!missing.isEmpty()) {
String err = shader.loc + " is missing: " + String.join(", ", missing);
throw new RuntimeException(err);
}
s = fillPrefixes(shader, s);
@ -56,14 +71,19 @@ public class InstancedArraysShaderTemplate {
StringBuffer out = new StringBuffer();
while (prefixMatches.find()) {
String structName = prefixMatches.group(1);
String prefix = prefixMatches.group(2);
String modifier = prefixMatches.group(2);
String prefix = prefixMatches.group(3);
TaggedStruct struct = shader.getStruct(structName);
StringBuilder builder = new StringBuilder();
for (String field : struct.fields.keySet()) {
for (Map.Entry<String, String> field : struct.fields.entrySet()) {
builder.append(modifier);
builder.append(' ');
builder.append(field.getValue());
builder.append(' ');
builder.append(prefix);
builder.append(field);
builder.append(field.getKey());
builder.append(";\n");
}

View file

@ -0,0 +1,33 @@
package com.jozufozu.flywheel.backend.loading;
import java.util.LinkedList;
public class ShaderTransformer {
private final LinkedList<ProcessingStage> stages = new LinkedList<>();
public ShaderTransformer() {
}
public ShaderTransformer pushStage(ProcessingStage stage) {
if (stage != null) {
stages.addLast(stage);
}
return this;
}
public ShaderTransformer prependStage(ProcessingStage stage) {
if (stage != null) {
stages.addFirst(stage);
}
return this;
}
public void transformSource(Shader shader) {
for (ProcessingStage stage : this.stages) {
stage.process(shader);
}
}
}

View file

@ -1,5 +1,3 @@
#version 110
#flwbuiltins
#[FLWFragment]

View file

@ -29,12 +29,11 @@ Raster FLWMain(Vertex v, Instance i) {
vec4 worldPos = i.transform * vec4(v.pos, 1.);
vec3 norm = i.normalMat * v.normal;
norm = normalize(norm);
FLWFinalizeWorldPos(worldPos);
FLWFinalizeNormal(norm);
norm = normalize(norm);
Raster r;
r.diffuse = diffuse(norm);
r.texCoords = v.texCoords;

View file

@ -1,12 +0,0 @@
#version 110
#flwbeginbody
#FLWPrefixFields(FLWFragment, varying __v2f_)
void main() {
FLWFragment f;
#FLWAssignFields(FLWFragment, f., __v2f_)
FLWMain(f);
}

View file

@ -1,19 +0,0 @@
#version 110
#flwbeginbody
#FLWPrefixFields(FLWVertexData, attribute __a_)
#FLWPrefixFields(FLWInstanceData, attribute __a_)
#FLWPrefixFields(FLWFragment, varying __v2f_)
void main() {
FLWVertexData v;
#FLWAssignFields(FLWVertexData, v., __a_)
FLWInstanceData i;
#FLWAssignFields(FLWInstanceData, i., __a_)
FLWFragment o = FLWMain(v, i);
#FLWAssignFields(FLWFragment, __v2f_, o.)
}

View file

@ -0,0 +1,12 @@
#version 110
#flwbeginbody
#FLWPrefixFields(FLWFragment, varying, v2f_)
void main() {
FLWFragment f;
#FLWAssignFields(FLWFragment, f., v2f_)
FLWMain(f);
}

View file

@ -0,0 +1,19 @@
#version 110
#flwbeginbody
#FLWPrefixFields(FLWVertexData, attribute, a_v_)
#FLWPrefixFields(FLWInstanceData, attribute, a_i_)
#FLWPrefixFields(FLWFragment, varying, v2f_)
void main() {
FLWVertexData v;
#FLWAssignFields(FLWVertexData, v., a_v_)
FLWInstanceData i;
#FLWAssignFields(FLWInstanceData, i., a_i_)
FLWFragment o = FLWMain(v, i);
#FLWAssignFields(FLWFragment, v2f_, o.)
}