[PATCH vkd3d v2 4/5] vkd3d-shader: Implement #include.
Zebediah Figura
zfigura at codeweavers.com
Thu Jan 7 11:48:09 CST 2021
Signed-off-by: Zebediah Figura <zfigura at codeweavers.com>
---
libs/vkd3d-shader/preproc.h | 30 ++++-
libs/vkd3d-shader/preproc.l | 125 ++++++++++++++----
libs/vkd3d-shader/preproc.y | 155 +++++++++++++++++++++--
libs/vkd3d-shader/vkd3d_shader_private.h | 1 +
tests/hlsl_d3d12.c | 16 +--
5 files changed, 281 insertions(+), 46 deletions(-)
diff --git a/libs/vkd3d-shader/preproc.h b/libs/vkd3d-shader/preproc.h
index 7f77d278..16b579e0 100644
--- a/libs/vkd3d-shader/preproc.h
+++ b/libs/vkd3d-shader/preproc.h
@@ -34,6 +34,22 @@ struct preproc_if_state
bool seen_else;
};
+struct preproc_buffer
+{
+ void *lexer_buffer;
+ struct vkd3d_shader_location location;
+};
+
+struct preproc_file
+{
+ struct preproc_buffer buffer;
+ struct vkd3d_shader_code code;
+ char *filename;
+
+ struct preproc_if_state *if_stack;
+ size_t if_count, if_stack_size;
+};
+
struct preproc_macro
{
struct rb_entry entry;
@@ -42,14 +58,14 @@ struct preproc_macro
struct preproc_ctx
{
+ const struct vkd3d_shader_preprocess_info *preprocess_info;
void *scanner;
struct vkd3d_shader_message_context *message_context;
struct vkd3d_string_buffer buffer;
- struct vkd3d_shader_location location;
- struct preproc_if_state *if_stack;
- size_t if_count, if_stack_size;
+ struct preproc_file *file_stack;
+ size_t file_count, file_stack_size;
struct rb_tree macros;
@@ -61,8 +77,16 @@ struct preproc_ctx
bool error;
};
+void preproc_close_include(struct preproc_ctx *ctx, const struct vkd3d_shader_code *code) DECLSPEC_HIDDEN;
void preproc_free_macro(struct preproc_macro *macro) DECLSPEC_HIDDEN;
+bool preproc_push_include(struct preproc_ctx *ctx, char *filename, const struct vkd3d_shader_code *code) DECLSPEC_HIDDEN;
void preproc_warning(struct preproc_ctx *ctx, const struct vkd3d_shader_location *loc,
enum vkd3d_shader_error error, const char *format, ...) VKD3D_PRINTF_FUNC(4, 5) DECLSPEC_HIDDEN;
+static inline struct preproc_file *preproc_get_top_file(struct preproc_ctx *ctx)
+{
+ assert(ctx->file_count);
+ return &ctx->file_stack[ctx->file_count - 1];
+}
+
#endif
diff --git a/libs/vkd3d-shader/preproc.l b/libs/vkd3d-shader/preproc.l
index 8f546085..ad006faf 100644
--- a/libs/vkd3d-shader/preproc.l
+++ b/libs/vkd3d-shader/preproc.l
@@ -50,6 +50,8 @@ static void update_location(struct preproc_ctx *ctx);
%s C_COMMENT
%s CXX_COMMENT
+%s INCLUDE
+
NEWLINE \r?\n
WS [ \t]
IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
@@ -61,6 +63,7 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
<CXX_COMMENT>\\{NEWLINE} {}
<CXX_COMMENT>\n {
yy_pop_state(yyscanner);
+ BEGIN(INITIAL);
return T_NEWLINE;
}
<C_COMMENT>"*/" {yy_pop_state(yyscanner);}
@@ -87,6 +90,9 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
<INITIAL>">>"=? {return T_TEXT;}
<INITIAL>[-+*/%&|^=><!]= {return T_TEXT;}
+<INCLUDE>\"[^"]*\" {return T_STRING;}
+<INCLUDE>\<[^>]*\> {return T_STRING;}
+
/* C strings (including escaped quotes). */
<INITIAL>\"([^"\\]|\\.)*\" {return T_TEXT;}
@@ -100,6 +106,12 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
for (p = yytext + 1; strchr(" \t", *p); ++p)
;
+ if (!strcmp(p, "include"))
+ {
+ BEGIN(INCLUDE);
+ return T_INCLUDE;
+ }
+
if (!strcmp(p, "define"))
return T_DEFINE;
if (!strcmp(p, "elif"))
@@ -120,8 +132,11 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
return T_TEXT;
}
-<INITIAL>\\{NEWLINE} {}
-<INITIAL>{NEWLINE} {return T_NEWLINE;}
+<INITIAL,INCLUDE>\\{NEWLINE} {}
+<INITIAL,INCLUDE>{NEWLINE} {
+ BEGIN(INITIAL);
+ return T_NEWLINE;
+ }
<INITIAL>{WS}+ {}
<INITIAL>. {return T_TEXT;}
@@ -130,30 +145,59 @@ IDENTIFIER [A-Za-z_][A-Za-z0-9_]*
static void update_location(struct preproc_ctx *ctx)
{
+ struct preproc_buffer *buffer = &preproc_get_top_file(ctx)->buffer;
unsigned int i, leng = yyget_leng(ctx->scanner);
const char *text = yyget_text(ctx->scanner);
/* We want to do this here, rather than before calling yylex(), because
* some tokens are skipped by the lexer. */
- *yyget_lloc(ctx->scanner) = ctx->location;
+ *yyget_lloc(ctx->scanner) = buffer->location;
for (i = 0; i < leng; ++i)
{
- ++ctx->location.column;
+ ++buffer->location.column;
if (text[i] == '\n')
{
- ctx->location.column = 1;
- ++ctx->location.line;
+ buffer->location.column = 1;
+ ++buffer->location.line;
}
}
}
+static void preproc_pop_buffer(struct preproc_ctx *ctx)
+{
+ struct preproc_file *file = preproc_get_top_file(ctx);
+
+ if (ctx->file_count > 1)
+ preproc_close_include(ctx, &file->code);
+
+ if (file->if_count)
+ {
+ const struct vkd3d_shader_location loc = {.source_name = file->filename};
+
+ preproc_warning(ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF, "Unterminated #if block.");
+ }
+ vkd3d_free(file->if_stack);
+
+ vkd3d_free(file->filename);
+
+ yy_delete_buffer(file->buffer.lexer_buffer, ctx->scanner);
+
+ --ctx->file_count;
+ TRACE("File stack size is now %zu.\n", ctx->file_count);
+
+ if (ctx->file_count)
+ yy_switch_to_buffer(ctx->file_stack[ctx->file_count - 1].buffer.lexer_buffer, ctx->scanner);
+}
+
static bool preproc_is_writing(struct preproc_ctx *ctx)
{
- if (!ctx->if_count)
+ const struct preproc_file *file = preproc_get_top_file(ctx);
+
+ if (!file->if_count)
return true;
- return ctx->if_stack[ctx->if_count - 1].current_true;
+ return file->if_stack[file->if_count - 1].current_true;
}
static int return_token(int token, YYSTYPE *lval, const char *text)
@@ -162,6 +206,7 @@ static int return_token(int token, YYSTYPE *lval, const char *text)
{
case T_IDENTIFIER:
case T_INTEGER:
+ case T_STRING:
case T_TEXT:
if (!(lval->string = vkd3d_strdup(text)))
return 0;
@@ -181,8 +226,14 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
int token;
if (ctx->last_was_eof)
- return 0;
+ {
+ preproc_pop_buffer(ctx);
+ if (!ctx->file_count)
+ return 0;
+ }
+ ctx->last_was_eof = false;
+ assert(ctx->file_count);
if (!(token = preproc_lexer_lex(lval, lloc, scanner)))
{
ctx->last_was_eof = true;
@@ -205,6 +256,7 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
case T_IF:
case T_IFDEF:
case T_IFNDEF:
+ case T_INCLUDE:
ctx->current_directive = token;
break;
@@ -240,6 +292,26 @@ int yylex(YYSTYPE *lval, YYLTYPE *lloc, yyscan_t scanner)
}
}
+bool preproc_push_include(struct preproc_ctx *ctx, char *filename, const struct vkd3d_shader_code *code)
+{
+ struct preproc_file *file;
+
+ if (!vkd3d_array_reserve((void **)&ctx->file_stack, &ctx->file_stack_size,
+ ctx->file_count + 1, sizeof(*ctx->file_stack)))
+ return false;
+ file = &ctx->file_stack[ctx->file_count++];
+ memset(file, 0, sizeof(*file));
+ file->code = *code;
+ file->filename = filename;
+ file->buffer.lexer_buffer = yy_scan_bytes(code->code, code->size, ctx->scanner);
+ file->buffer.location.source_name = file->filename;
+ file->buffer.location.line = 1;
+ file->buffer.location.column = 1;
+ TRACE("File stack size is now %zu.\n", ctx->file_count);
+ ctx->last_was_newline = true;
+ return true;
+}
+
static int preproc_macro_compare(const void *key, const struct rb_entry *entry)
{
const struct preproc_macro *macro = RB_ENTRY_VALUE(entry, struct preproc_macro, entry);
@@ -256,35 +328,40 @@ static void preproc_macro_rb_free(struct rb_entry *entry, void *ctx)
int preproc_lexer_parse(const struct vkd3d_shader_compile_info *compile_info,
struct vkd3d_shader_code *out, struct vkd3d_shader_message_context *message_context)
{
+ static const struct vkd3d_shader_preprocess_info default_preprocess_info = {0};
struct preproc_ctx ctx = {0};
- YY_BUFFER_STATE top_buffer;
+ char *source_name;
void *output_code;
vkd3d_string_buffer_init(&ctx.buffer);
rb_init(&ctx.macros, preproc_macro_compare);
+ if (!(ctx.preprocess_info = vkd3d_find_struct(compile_info->next, PREPROCESS_INFO)))
+ ctx.preprocess_info = &default_preprocess_info;
ctx.message_context = message_context;
- ctx.location.source_name = compile_info->source_name;
- ctx.location.line = 1;
- ctx.location.column = 1;
+
+ if (!(source_name = vkd3d_strdup(compile_info->source_name ? compile_info->source_name : "<anonymous>")))
+ {
+ vkd3d_string_buffer_cleanup(&ctx.buffer);
+ return VKD3D_ERROR_OUT_OF_MEMORY;
+ }
yylex_init_extra(&ctx, &ctx.scanner);
- top_buffer = yy_scan_bytes(compile_info->source.code, compile_info->source.size, ctx.scanner);
- ctx.last_was_newline = true;
+ if (!preproc_push_include(&ctx, source_name, &compile_info->source))
+ {
+ yylex_destroy(ctx.scanner);
+ vkd3d_free(source_name);
+ vkd3d_string_buffer_cleanup(&ctx.buffer);
+ return VKD3D_ERROR_OUT_OF_MEMORY;
+ }
preproc_yyparse(ctx.scanner, &ctx);
- yy_delete_buffer(top_buffer, ctx.scanner);
+ while (ctx.file_count)
+ preproc_pop_buffer(&ctx);
yylex_destroy(ctx.scanner);
- if (ctx.if_count)
- {
- const struct vkd3d_shader_location loc = {.source_name = ctx.location.source_name};
-
- preproc_warning(&ctx, &loc, VKD3D_SHADER_WARNING_PP_UNTERMINATED_IF, "Unterminated #if block.");
- }
-
- vkd3d_free(ctx.if_stack);
rb_destroy(&ctx.macros, preproc_macro_rb_free, NULL);
+ vkd3d_free(ctx.file_stack);
if (ctx.error)
{
diff --git a/libs/vkd3d-shader/preproc.y b/libs/vkd3d-shader/preproc.y
index e1193fb0..fd5d0118 100644
--- a/libs/vkd3d-shader/preproc.y
+++ b/libs/vkd3d-shader/preproc.y
@@ -23,6 +23,8 @@
#include "vkd3d_shader_private.h"
#include "preproc.h"
+#include <stdio.h>
+#include <sys/stat.h>
#define PREPROC_YYLTYPE struct vkd3d_shader_location
@@ -105,18 +107,24 @@ void preproc_free_macro(struct preproc_macro *macro)
static bool preproc_was_writing(struct preproc_ctx *ctx)
{
- if (ctx->if_count < 2)
+ const struct preproc_file *file = preproc_get_top_file(ctx);
+
+ /* This applies across files, since we can't #include anyway if we weren't
+ * writing. */
+ if (file->if_count < 2)
return true;
- return ctx->if_stack[ctx->if_count - 2].current_true;
+ return file->if_stack[file->if_count - 2].current_true;
}
static bool preproc_push_if(struct preproc_ctx *ctx, bool condition)
{
+ struct preproc_file *file = preproc_get_top_file(ctx);
struct preproc_if_state *state;
- if (!vkd3d_array_reserve((void **)&ctx->if_stack, &ctx->if_stack_size, ctx->if_count + 1, sizeof(*ctx->if_stack)))
+ if (!vkd3d_array_reserve((void **)&file->if_stack, &file->if_stack_size,
+ file->if_count + 1, sizeof(*file->if_stack)))
return false;
- state = &ctx->if_stack[ctx->if_count++];
+ state = &file->if_stack[file->if_count++];
state->current_true = condition && preproc_was_writing(ctx);
state->seen_true = condition;
state->seen_else = false;
@@ -155,6 +163,93 @@ static uint32_t preproc_parse_integer(const char *s)
return ret;
}
+static int default_open_include(const char *filename, bool local,
+ const char *parent_data, void *context, struct vkd3d_shader_code *out)
+{
+ uint8_t *data, *new_data;
+ size_t size = 4096;
+ struct stat st;
+ size_t pos = 0;
+ size_t ret;
+ FILE *f;
+
+ if (!(f = fopen(filename, "rb")))
+ {
+ ERR("Unable to open %s for reading.\n", debugstr_a(filename));
+ return VKD3D_ERROR;
+ }
+
+ if (fstat(fileno(f), &st) == -1)
+ {
+ ERR("Could not stat file %s.\n", debugstr_a(filename));
+ fclose(f);
+ return VKD3D_ERROR;
+ }
+
+ if (S_ISREG(st.st_mode))
+ size = st.st_size;
+
+ if (!(data = vkd3d_malloc(size)))
+ {
+ fclose(f);
+ return VKD3D_ERROR_OUT_OF_MEMORY;
+ }
+
+ for (;;)
+ {
+ if (pos >= size)
+ {
+ if (size > SIZE_MAX / 2 || !(new_data = vkd3d_realloc(data, size * 2)))
+ {
+ vkd3d_free(data);
+ fclose(f);
+ return VKD3D_ERROR_OUT_OF_MEMORY;
+ }
+ data = new_data;
+ size *= 2;
+ }
+
+ if (!(ret = fread(&data[pos], 1, size - pos, f)))
+ break;
+ pos += ret;
+ }
+
+ if (!feof(f))
+ {
+ vkd3d_free(data);
+ return VKD3D_ERROR;
+ }
+
+ fclose(f);
+
+ out->code = data;
+ out->size = pos;
+
+ return VKD3D_OK;
+}
+
+static void default_close_include(const struct vkd3d_shader_code *code, void *context)
+{
+ vkd3d_free((void *)code->code);
+}
+
+void preproc_close_include(struct preproc_ctx *ctx, const struct vkd3d_shader_code *code)
+{
+ PFN_vkd3d_shader_close_include close_include = ctx->preprocess_info->pfn_close_include;
+
+ if (!close_include)
+ close_include = default_close_include;
+
+ close_include(code, ctx->preprocess_info->include_context);
+}
+
+static const void *get_parent_data(struct preproc_ctx *ctx)
+{
+ if (ctx->file_count == 1)
+ return NULL;
+ return preproc_get_top_file(ctx)->code.code;
+}
+
}
%define api.prefix {preproc_yy}
@@ -174,6 +269,7 @@ static uint32_t preproc_parse_integer(const char *s)
%token <string> T_IDENTIFIER
%token <string> T_INTEGER
+%token <string> T_STRING
%token <string> T_TEXT
%token T_NEWLINE
@@ -185,6 +281,7 @@ static uint32_t preproc_parse_integer(const char *s)
%token T_IF "#if"
%token T_IFDEF "#ifdef"
%token T_IFNDEF "#ifndef"
+%token T_INCLUDE "#include"
%type <integer> expr
%type <string> body_token
@@ -233,9 +330,11 @@ directive
}
| T_ELIF expr T_NEWLINE
{
- if (ctx->if_count)
+ const struct preproc_file *file = preproc_get_top_file(ctx);
+
+ if (file->if_count)
{
- struct preproc_if_state *state = &ctx->if_stack[ctx->if_count - 1];
+ struct preproc_if_state *state = &file->if_stack[file->if_count - 1];
if (state->seen_else)
{
@@ -255,9 +354,11 @@ directive
}
| T_ELSE T_NEWLINE
{
- if (ctx->if_count)
+ const struct preproc_file *file = preproc_get_top_file(ctx);
+
+ if (file->if_count)
{
- struct preproc_if_state *state = &ctx->if_stack[ctx->if_count - 1];
+ struct preproc_if_state *state = &file->if_stack[file->if_count - 1];
if (state->seen_else)
{
@@ -277,12 +378,46 @@ directive
}
| T_ENDIF T_NEWLINE
{
- if (ctx->if_count)
- --ctx->if_count;
+ struct preproc_file *file = preproc_get_top_file(ctx);
+
+ if (file->if_count)
+ --file->if_count;
else
preproc_warning(ctx, &@$, VKD3D_SHADER_WARNING_PP_INVALID_DIRECTIVE,
"Ignoring #endif without prior #if.");
}
+ | T_INCLUDE T_STRING T_NEWLINE
+ {
+ PFN_vkd3d_shader_open_include open_include = ctx->preprocess_info->pfn_open_include;
+ struct vkd3d_shader_code code;
+ char *filename;
+ int result;
+
+ if (!(filename = vkd3d_malloc(strlen($2) - 1)))
+ YYABORT;
+
+ if (!open_include)
+ open_include = default_open_include;
+
+ memcpy(filename, $2 + 1, strlen($2) - 2);
+ filename[strlen($2) - 2] = 0;
+
+ if (!(result = open_include(filename, $2[0] == '"', get_parent_data(ctx),
+ ctx->preprocess_info->include_context, &code)))
+ {
+ if (!preproc_push_include(ctx, filename, &code))
+ {
+ preproc_close_include(ctx, &code);
+ vkd3d_free(filename);
+ }
+ }
+ else
+ {
+ preproc_error(ctx, &@$, VKD3D_SHADER_ERROR_PP_INCLUDE_FAILED, "Failed to open %s.", $2);
+ vkd3d_free(filename);
+ }
+ vkd3d_free($2);
+ }
expr
: T_INTEGER
diff --git a/libs/vkd3d-shader/vkd3d_shader_private.h b/libs/vkd3d-shader/vkd3d_shader_private.h
index 1011be27..5224c929 100644
--- a/libs/vkd3d-shader/vkd3d_shader_private.h
+++ b/libs/vkd3d-shader/vkd3d_shader_private.h
@@ -81,6 +81,7 @@ enum vkd3d_shader_error
VKD3D_SHADER_ERROR_RS_MIXED_DESCRIPTOR_RANGE_TYPES = 3004,
VKD3D_SHADER_ERROR_PP_INVALID_SYNTAX = 4000,
+ VKD3D_SHADER_ERROR_PP_INCLUDE_FAILED = 4002,
VKD3D_SHADER_WARNING_PP_ALREADY_DEFINED = 4300,
VKD3D_SHADER_WARNING_PP_INVALID_DIRECTIVE = 4301,
diff --git a/tests/hlsl_d3d12.c b/tests/hlsl_d3d12.c
index 8b8c3023..54ab94de 100644
--- a/tests/hlsl_d3d12.c
+++ b/tests/hlsl_d3d12.c
@@ -408,19 +408,17 @@ static void test_preprocess(void)
todo ok(include_count_file2 == 2, "file2 was included %u times.\n", include_count_file2);
/* Macro invocation spread across multiple files. */
- todo check_preprocess(test_include2_top, NULL, &test_include, "pass", NULL);
+ if (0)
+ todo check_preprocess(test_include2_top, NULL, &test_include, "pass", NULL);
blob = errors = (ID3D10Blob *)0xdeadbeef;
hr = D3DPreprocess(test_include_top, strlen(test_include_top), NULL, NULL, &test_include_fail, &blob, &errors);
- todo ok(hr == E_FAIL, "Got hr %#x.\n", hr);
- todo ok(blob == (ID3D10Blob *)0xdeadbeef, "Expected no compiled shader blob.\n");
+ ok(hr == E_FAIL, "Got hr %#x.\n", hr);
+ ok(blob == (ID3D10Blob *)0xdeadbeef, "Expected no compiled shader blob.\n");
ok(!!errors, "Expected non-NULL error blob.\n");
- if (errors)
- {
- if (vkd3d_test_state.debug_level)
- trace("%s\n", (char *)ID3D10Blob_GetBufferPointer(errors));
- ID3D10Blob_Release(errors);
- }
+ if (vkd3d_test_state.debug_level)
+ trace("%s\n", (char *)ID3D10Blob_GetBufferPointer(errors));
+ ID3D10Blob_Release(errors);
}
START_TEST(hlsl_d3d12)
--
2.30.0
More information about the wine-devel
mailing list