Multiple payload types in tokens

Instead of just string (and empty strings for tokens without payload)

1 files changed, 32 insertions, 22 deletions

diff --git a/gramlib/grammar.mli b/gramlib/grammar.mli
index 453ec85187..ec4ec62409 100644
--- a/gramlib/grammar.mli
+++ b/gramlib/grammar.mli
@@ -15,13 +15,14 @@
        rule "an entry cannot call an entry of another grammar" by
        normal OCaml typing. *)
 
-module type GLexerType = sig type te val lexer : te Plexing.lexer end
+module type GLexerType = Plexing.Lexer
    (** The input signature for the functor [Grammar.GMake]: [te] is the
        type of the tokens. *)
 
 module type S =
   sig
     type te
+    type 'c pattern
     type parsable
     val parsable : ?loc:Loc.t -> char Stream.t -> parsable
     val tokens : string -> (string option * int) list
@@ -35,29 +36,37 @@ module type S =
         val parse_token_stream : 'a e -> te Stream.t -> 'a
         val print : Format.formatter -> 'a e -> unit
       end
-    type ('self, 'a) ty_symbol
-    type ('self, 'f, 'r) ty_rule
+    type ty_norec = TyNoRec
+    type ty_mayrec = TyMayRec
+    type ('self, 'trec, 'a) ty_symbol
+    type ('self, 'trec, 'f, 'r) ty_rule
+    type 'a ty_rules
     type 'a ty_production
-    val s_nterm : 'a Entry.e -> ('self, 'a) ty_symbol
-    val s_nterml : 'a Entry.e -> string -> ('self, 'a) ty_symbol
-    val s_list0 : ('self, 'a) ty_symbol -> ('self, 'a list) ty_symbol
+    val s_nterm : 'a Entry.e -> ('self, ty_norec, 'a) ty_symbol
+    val s_nterml : 'a Entry.e -> string -> ('self, ty_norec, 'a) ty_symbol
+    val s_list0 : ('self, 'trec, 'a) ty_symbol -> ('self, 'trec, 'a list) ty_symbol
     val s_list0sep :
-      ('self, 'a) ty_symbol -> ('self, 'b) ty_symbol -> bool ->
-        ('self, 'a list) ty_symbol
-    val s_list1 : ('self, 'a) ty_symbol -> ('self, 'a list) ty_symbol
+      ('self, 'trec, 'a) ty_symbol -> ('self, ty_norec, 'b) ty_symbol -> bool ->
+        ('self, 'trec, 'a list) ty_symbol
+    val s_list1 : ('self, 'trec, 'a) ty_symbol -> ('self, 'trec, 'a list) ty_symbol
     val s_list1sep :
-      ('self, 'a) ty_symbol -> ('self, 'b) ty_symbol -> bool ->
-        ('self, 'a list) ty_symbol
-    val s_opt : ('self, 'a) ty_symbol -> ('self, 'a option) ty_symbol
-    val s_self : ('self, 'self) ty_symbol
-    val s_next : ('self, 'self) ty_symbol
-    val s_token : Plexing.pattern -> ('self, string) ty_symbol
-    val s_rules : warning:(string -> unit) option -> 'a ty_production list -> ('self, 'a) ty_symbol
-    val r_stop : ('self, 'r, 'r) ty_rule
+      ('self, 'trec, 'a) ty_symbol -> ('self, ty_norec, 'b) ty_symbol -> bool ->
+        ('self, 'trec, 'a list) ty_symbol
+    val s_opt : ('self, 'trec, 'a) ty_symbol -> ('self, 'trec, 'a option) ty_symbol
+    val s_self : ('self, ty_mayrec, 'self) ty_symbol
+    val s_next : ('self, ty_mayrec, 'self) ty_symbol
+    val s_token : 'c pattern -> ('self, ty_norec, 'c) ty_symbol
+    val s_rules : warning:(string -> unit) option -> 'a ty_rules list -> ('self, ty_norec, 'a) ty_symbol
+
+    val r_stop : ('self, ty_norec, 'r, 'r) ty_rule
     val r_next :
-      ('self, 'a, 'r) ty_rule -> ('self, 'b) ty_symbol ->
-        ('self, 'b -> 'a, 'r) ty_rule
-    val production : ('a, 'f, Loc.t -> 'a) ty_rule * 'f -> 'a ty_production
+      ('self, _, 'a, 'r) ty_rule -> ('self, _, 'b) ty_symbol ->
+        ('self, ty_mayrec, 'b -> 'a, 'r) ty_rule
+    val r_next_norec :
+      ('self, ty_norec, 'a, 'r) ty_rule -> ('self, ty_norec, 'b) ty_symbol ->
+        ('self, ty_norec, 'b -> 'a, 'r) ty_rule
+    val rules : (_, ty_norec, 'f, Loc.t -> 'a) ty_rule * 'f -> 'a ty_rules
+    val production : ('a, _, 'f, Loc.t -> 'a) ty_rule * 'f -> 'a ty_production
 
     module Unsafe :
       sig
@@ -68,7 +77,7 @@ module type S =
         (string option * Gramext.g_assoc option * 'a ty_production list)
           list ->
         unit
-    val safe_delete_rule : 'a Entry.e -> ('a, 'f, 'r) ty_rule -> unit
+    val safe_delete_rule : 'a Entry.e -> ('a, _, 'f, 'r) ty_rule -> unit
   end
    (** Signature type of the functor [Grammar.GMake]. The types and
        functions are almost the same than in generic interface, but:
@@ -80,4 +89,5 @@ module type S =
          type (instead of (string * string)); the module parameter
          must specify a way to show them as (string * string) *)
 
-module GMake (L : GLexerType) : S with type te = L.te
+module GMake (L : GLexerType) :
+  S with type te = L.te and type 'c pattern = 'c L.pattern