Merge PR #9733: [lexer] keyword protected quotation token for arbitrary text

Ack-by: Zimmi48
Ack-by: ejgallego
Ack-by: gares
Reviewed-by: ppedrot

4 files changed, 9 insertions, 9 deletions

diff --git a/gramlib/grammar.ml b/gramlib/grammar.ml
index 74350c4f15..9eebe7a1e2 100644
--- a/gramlib/grammar.ml
+++ b/gramlib/grammar.ml
@@ -16,7 +16,7 @@ module type S =
     type te
     type parsable
     val parsable : ?loc:Loc.t -> char Stream.t -> parsable
-    val tokens : string -> (string * int) list
+    val tokens : string -> (string option * int) list
     module Entry :
       sig
         type 'a e
@@ -195,7 +195,7 @@ let is_before : type s1 s2 a1 a2. (s1, a1) ty_symbol -> (s2, a2) ty_symbol -> bo
   match s1, s2 with
     Stoken ("ANY", _), _ -> false
   | _, Stoken ("ANY", _) -> true
-  | Stoken (_, s), Stoken (_, "") when s <> "" -> true
+  | Stoken (_, Some _), Stoken (_, None) -> true
   | Stoken _, Stoken _ -> false
   | Stoken _, _ -> true
   | _ -> false
@@ -683,7 +683,7 @@ let rec print_symbol : type s r. formatter -> (s, r) ty_symbol -> unit =
       fprintf ppf "LIST1 %a SEP %a%s" print_symbol1 s print_symbol1 t
         (if osep then " OPT_SEP" else "")
   | Sopt s -> fprintf ppf "OPT %a" print_symbol1 s
-  | Stoken (con, prm) when con <> "" && prm <> "" ->
+  | Stoken (con, Some prm) when con <> "" ->
       fprintf ppf "%s@ %a" con print_str prm
   | Snterml (e, l) ->
       fprintf ppf "%s%s@ LEVEL@ %a" e.ename ""
@@ -695,8 +695,8 @@ and print_symbol1 : type s r. formatter -> (s, r) ty_symbol -> unit =
   | Snterm e -> fprintf ppf "%s%s" e.ename ""
   | Sself -> pp_print_string ppf "SELF"
   | Snext -> pp_print_string ppf "NEXT"
-  | Stoken ("", s) -> print_str ppf s
-  | Stoken (con, "") -> pp_print_string ppf con
+  | Stoken ("", Some s) -> print_str ppf s
+  | Stoken (con, None) -> pp_print_string ppf con
   | Stree t -> print_level ppf pp_print_space (flatten_tree t)
   | s ->
       fprintf ppf "(%a)" print_symbol s
diff --git a/gramlib/grammar.mli b/gramlib/grammar.mli
index 7cb7a92f85..453ec85187 100644
--- a/gramlib/grammar.mli
+++ b/gramlib/grammar.mli
@@ -24,7 +24,7 @@ module type S =
     type te
     type parsable
     val parsable : ?loc:Loc.t -> char Stream.t -> parsable
-    val tokens : string -> (string * int) list
+    val tokens : string -> (string option * int) list
     module Entry :
       sig
         type 'a e
diff --git a/gramlib/plexing.ml b/gramlib/plexing.ml
index c294923a85..6da06f147f 100644
--- a/gramlib/plexing.ml
+++ b/gramlib/plexing.ml
@@ -2,7 +2,7 @@
 (* plexing.ml,v *)
 (* Copyright (c) INRIA 2007-2017 *)
 
-type pattern = string * string
+type pattern = string * string option
 
 type location_function = int -> Loc.t
 type 'te lexer_func = ?loc:Loc.t -> char Stream.t -> 'te Stream.t * location_function
diff --git a/gramlib/plexing.mli b/gramlib/plexing.mli
index f6e4d96b80..26443df026 100644
--- a/gramlib/plexing.mli
+++ b/gramlib/plexing.mli
@@ -8,13 +8,13 @@
    grammars (see module [Grammar]). It also provides some useful functions
    to create lexers. *)
 
-type pattern = string * string
+type pattern = string * string option
     (* Type for values used by the generated code of the EXTEND
        statement to represent terminals in entry rules.
 -      The first string is the constructor name (must start with
        an uppercase character). When it is empty, the second string
        is supposed to be a keyword.
--      The second string is the constructor parameter. Empty if it
+-      The second component is the constructor parameter. None if it
        has no parameter (corresponding to the 'wildcard' pattern).
 -      The way tokens patterns are interpreted to parse tokens is done
        by the lexer, function [tok_match] below. *)