From 242fd9baa58e3c6d2043d68b7deddcbfb68fb4a9 Mon Sep 17 00:00:00 2001
From: omicron <omicron.me@protonmail.com>
Date: Wed, 16 Apr 2025 12:34:44 +0200
Subject: [PATCH] Fix grammar not being able to disambiguate some instructions

When two identifiers follow eachother it could be two instruction
mnemonics or one instruction mnemonic and one operand. To fix this
TOKEN_NEWLINE has been reintroduced as a semantic token. The grammar has
been changed to allow empty statements and every instruction and
directive has to end in a newline. Labels do not have to end in a
newline.

In addition to updating the grammar, the implementation of tokenlist,
ast and parser has been updated to reflect these changes.
---
 doc/parser_grammar.txt  | 6 +++---
 src/ast.c               | 5 ++++-
 src/ast.h               | 1 +
 src/parser/parser.c     | 8 +++++---
 src/parser/primitives.c | 4 ++++
 src/parser/primitives.h | 1 +
 src/tokenlist.c         | 1 -
 7 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/doc/parser_grammar.txt b/doc/parser_grammar.txt
index e7e9315..9a651b9 100644
--- a/doc/parser_grammar.txt
+++ b/doc/parser_grammar.txt
@@ -1,13 +1,13 @@
 <program>   ::= <statement>*
-<statement> ::= <label> | <directive> | <instruction>
+<statement> ::= <label> | <directive> | <instruction> | <newline>
 
 <label> ::= <identifier> <colon>
 
-<directive> ::= <dot> <section_directive>
+<directive> ::= <dot> <section_directive> <newline>
 
 <section_directive> ::= "section" <identifier>
 
-<instruction> ::= <identifier> <operands>
+<instruction> ::= <identifier> <operands> <newline>
 
 <operands> ::= <operand> ( <comma> <operand> )*
 
diff --git a/src/ast.c b/src/ast.c
index 88caf60..5b93585 100644
--- a/src/ast.c
+++ b/src/ast.c
@@ -157,6 +157,8 @@ const char *ast_node_id_to_cstr(node_id_t id) {
         return "NODE_ASTERISK";
     case NODE_DOT:
         return "NODE_DOT";
+    case NODE_NEWLINE:
+        return "NODE_NEWLINE";
     }
     assert(!"Unreachable, weird node id" && id);
     __builtin_unreachable();
@@ -172,7 +174,8 @@ static void ast_node_print_internal(ast_node_t *node, int indent) {
     }
     printf("%s", ast_node_id_to_cstr(node->id));
 
-    if (node->token_entry && node->token_entry->token.value) {
+    if (node->token_entry && node->token_entry->token.value &&
+        node->id != NODE_NEWLINE) {
         printf(" \"%s\"", node->token_entry->token.value);
     }
     printf("\n");
diff --git a/src/ast.h b/src/ast.h
index fb6371c..dafd736 100644
--- a/src/ast.h
+++ b/src/ast.h
@@ -50,6 +50,7 @@ typedef enum node_id {
     NODE_MINUS,
     NODE_ASTERISK,
     NODE_DOT,
+    NODE_NEWLINE,
 } node_id_t;
 
 typedef struct ast_node ast_node_t;
diff --git a/src/parser/parser.c b/src/parser/parser.c
index 5f96681..d6d4a18 100644
--- a/src/parser/parser.c
+++ b/src/parser/parser.c
@@ -120,18 +120,20 @@ parse_result_t parse_section_directive(tokenlist_entry_t *current) {
 }
 
 parse_result_t parse_directive(tokenlist_entry_t *current) {
-    parser_t parsers[] = {parse_dot, parse_section_directive, nullptr};
+    parser_t parsers[] = {parse_dot, parse_section_directive, parse_newline,
+                          nullptr};
     return parse_consecutive(current, NODE_DIRECTIVE, parsers);
 }
 
 parse_result_t parse_instruction(tokenlist_entry_t *current) {
-    parser_t parsers[] = {parse_identifier, parse_operands, nullptr};
+    parser_t parsers[] = {parse_identifier, parse_operands, parse_newline,
+                          nullptr};
     return parse_consecutive(current, NODE_INSTRUCTION, parsers);
 }
 
 parse_result_t parse_statement(tokenlist_entry_t *current) {
     parser_t parsers[] = {parse_label, parse_directive, parse_instruction,
-                          nullptr};
+                          parse_newline, nullptr};
     return parse_any(current, parsers);
 }
 
diff --git a/src/parser/primitives.c b/src/parser/primitives.c
index 7e89aa9..c93231e 100644
--- a/src/parser/primitives.c
+++ b/src/parser/primitives.c
@@ -62,6 +62,10 @@ parse_result_t parse_dot(tokenlist_entry_t *current) {
     return parse_token(current, TOKEN_DOT, NODE_DOT, nullptr);
 }
 
+parse_result_t parse_newline(tokenlist_entry_t *current) {
+    return parse_token(current, TOKEN_NEWLINE, NODE_NEWLINE, nullptr);
+}
+
 parse_result_t parse_label_reference(tokenlist_entry_t *current) {
     return parse_token(current, TOKEN_IDENTIFIER, NODE_LABEL_REFERENCE,
                        nullptr);
diff --git a/src/parser/primitives.h b/src/parser/primitives.h
index e6df83f..46ed4ef 100644
--- a/src/parser/primitives.h
+++ b/src/parser/primitives.h
@@ -18,6 +18,7 @@ parse_result_t parse_plus(tokenlist_entry_t *current);
 parse_result_t parse_minus(tokenlist_entry_t *current);
 parse_result_t parse_asterisk(tokenlist_entry_t *current);
 parse_result_t parse_dot(tokenlist_entry_t *current);
+parse_result_t parse_newline(tokenlist_entry_t *current);
 parse_result_t parse_label_reference(tokenlist_entry_t *current);
 
 /* These are "primitives" with a different name and some extra validation on top
diff --git a/src/tokenlist.c b/src/tokenlist.c
index 84c2c39..4e33689 100644
--- a/src/tokenlist.c
+++ b/src/tokenlist.c
@@ -86,7 +86,6 @@ bool is_trivia(tokenlist_entry_t *trivia) {
     switch (trivia->token.id) {
     case TOKEN_WHITESPACE:
     case TOKEN_COMMENT:
-    case TOKEN_NEWLINE:
         return true;
     default:
         return false;