[parsing] Split Tok.t into Tok.t and Tok.pattern

Tokens were having a double role:
- the output of the lexer
- the items of grammar entries, especially terminals

Now tokens are the output of the lexer, and this paves the way for
using a richer data type, eg including Loc.t

Patterns, as in Plexing.pattern, only represent patterns (for tokens)
and now have a bit more structure (eg the wildcard is represented
as None, not as "", while a regular pattern for "x" as Some "x")

4 files changed, 8 insertions, 8 deletions

diff --git a/gramlib/grammar.ml b/gramlib/grammar.ml
index 74350c4f15..9eebe7a1e2 100644
--- a/gramlib/grammar.ml
+++ b/gramlib/grammar.ml
@@ -16,7 +16,7 @@ module type S =
     type te
     type parsable
     val parsable : ?loc:Loc.t -> char Stream.t -> parsable
-    val tokens : string -> (string * int) list
+    val tokens : string -> (string option * int) list
     module Entry :
       sig
         type 'a e
@@ -195,7 +195,7 @@ let is_before : type s1 s2 a1 a2. (s1, a1) ty_symbol -> (s2, a2) ty_symbol -> bo
   match s1, s2 with
     Stoken ("ANY", _), _ -> false
   | _, Stoken ("ANY", _) -> true
-  | Stoken (_, s), Stoken (_, "") when s <> "" -> true
+  | Stoken (_, Some _), Stoken (_, None) -> true
   | Stoken _, Stoken _ -> false
   | Stoken _, _ -> true
   | _ -> false
@@ -683,7 +683,7 @@ let rec print_symbol : type s r. formatter -> (s, r) ty_symbol -> unit =
       fprintf ppf "LIST1 %a SEP %a%s" print_symbol1 s print_symbol1 t
         (if osep then " OPT_SEP" else "")
   | Sopt s -> fprintf ppf "OPT %a" print_symbol1 s
-  | Stoken (con, prm) when con <> "" && prm <> "" ->
+  | Stoken (con, Some prm) when con <> "" ->
       fprintf ppf "%s@ %a" con print_str prm
   | Snterml (e, l) ->
       fprintf ppf "%s%s@ LEVEL@ %a" e.ename ""
@@ -695,8 +695,8 @@ and print_symbol1 : type s r. formatter -> (s, r) ty_symbol -> unit =
   | Snterm e -> fprintf ppf "%s%s" e.ename ""
   | Sself -> pp_print_string ppf "SELF"
   | Snext -> pp_print_string ppf "NEXT"
-  | Stoken ("", s) -> print_str ppf s
-  | Stoken (con, "") -> pp_print_string ppf con
+  | Stoken ("", Some s) -> print_str ppf s
+  | Stoken (con, None) -> pp_print_string ppf con
   | Stree t -> print_level ppf pp_print_space (flatten_tree t)
   | s ->
       fprintf ppf "(%a)" print_symbol s
diff --git a/gramlib/grammar.mli b/gramlib/grammar.mli
index 7cb7a92f85..453ec85187 100644
--- a/gramlib/grammar.mli
+++ b/gramlib/grammar.mli
@@ -24,7 +24,7 @@ module type S =
     type te
     type parsable
     val parsable : ?loc:Loc.t -> char Stream.t -> parsable
-    val tokens : string -> (string * int) list
+    val tokens : string -> (string option * int) list
     module Entry :
       sig
         type 'a e
diff --git a/gramlib/plexing.ml b/gramlib/plexing.ml
index c294923a85..6da06f147f 100644
--- a/gramlib/plexing.ml
+++ b/gramlib/plexing.ml
@@ -2,7 +2,7 @@
 (* plexing.ml,v *)
 (* Copyright (c) INRIA 2007-2017 *)
 
-type pattern = string * string
+type pattern = string * string option
 
 type location_function = int -> Loc.t
 type 'te lexer_func = ?loc:Loc.t -> char Stream.t -> 'te Stream.t * location_function
diff --git a/gramlib/plexing.mli b/gramlib/plexing.mli
index f6e4d96b80..48722e01be 100644
--- a/gramlib/plexing.mli
+++ b/gramlib/plexing.mli
@@ -8,7 +8,7 @@
    grammars (see module [Grammar]). It also provides some useful functions
    to create lexers. *)
 
-type pattern = string * string
+type pattern = string * string option
     (* Type for values used by the generated code of the EXTEND
        statement to represent terminals in entry rules.
 -      The first string is the constructor name (must start with